-- Name: czx_info_handler(numeric, numeric, numeric, character varying, character varying, character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.czx_info_handler(jobid numeric, messageid numeric, messageline numeric, messageprocedure character varying, infomessage character varying, stepnumber character varying) RETURNS numeric
    LANGUAGE plpgsql SECURITY DEFINER
    SET search_path TO 'tm_cz', 'pg_temp'
    AS $$
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/

declare

  databaseName VARCHAR(100);
  rtnCd			numeric;
BEGIN

	select 
    database_name INTO databasename
  from 
    tm_cz.cz_job_master 
	where 
    job_id=jobID;
    
  select tm_cz.czx_write_audit( jobID, databaseName, messageProcedure, 'Step contains more details', 0, stepNumber, 'Information' ) into rtnCd;

  select tm_cz.czx_write_info(jobID, messageID, messageLine, messageProcedure, infoMessage ) into rtnCd;
  return rtnCd;
  
END;
$$;


ALTER FUNCTION tm_cz.czx_info_handler(jobid numeric, messageid numeric, messageline numeric, messageprocedure character varying, infomessage character varying, stepnumber character varying) OWNER TO tm_cz;

--
-- Name: czx_percentile_cont(real[], real); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.czx_percentile_cont(myarray real[], percentile real) RETURNS real
    LANGUAGE plpgsql IMMUTABLE SECURITY DEFINER
    SET search_path TO 'tm_cz', 'tm_lz', 'tm_wz', 'i2b2demodata', 'i2b2metadata', 'deapp', 'pg_temp'
    AS $$   
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/ 
DECLARE   
  ary_cnt INTEGER;   
  row_num real;   
  crn real;   
  frn real;   
  calc_result real;   
  new_array real[];   
BEGIN   
  ary_cnt = array_length(myarray,1);   
  row_num = 1 + ( percentile * ( ary_cnt - 1 ));   
  new_array = tm_cz.czx_array_sort(myarray);   
     
  crn = ceiling(row_num);   
  frn = floor(row_num);   
         
      if crn = frn and frn = row_num then   
    calc_result = new_array[row_num];   
  else   
    calc_result = (crn - row_num) * new_array[frn]   
            + (row_num - frn) * new_array[crn];   
  end if;   
     
  RETURN calc_result;   
END;   
$$;


ALTER FUNCTION tm_cz.czx_percentile_cont(myarray real[], percentile real) OWNER TO tm_cz;

--
-- Name: czx_start_audit(character varying, character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.czx_start_audit(jobname character varying, databasename character varying) RETURNS numeric
    LANGUAGE plpgsql SECURITY DEFINER
    SET search_path TO 'tm_cz', 'pg_temp'
    AS $$
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
declare
	rtnCd	integer;
	jobId	numeric;
BEGIN
	begin
		insert into tm_cz.cz_job_master
			(start_date, 
			active, 
			database_name,
			job_name,
			job_status) 
		VALUES(
			CLOCK_TIMESTAMP(),
			'Y', 
			databaseName,
			jobName,
			'Running')
	  RETURNING job_id INTO jobID;
	end;
  
  return jobID;
  
  exception 
	when OTHERS then
		select tm_cz.czx_write_error(jobId,'0',SQLERRML,SQLSTATE,SQLERRM) into rtnCd;
		return -16;

END;

$$;


ALTER FUNCTION tm_cz.czx_start_audit(jobname character varying, databasename character varying) OWNER TO tm_cz;

--
-- Name: czx_table_index_maint(character varying, character varying, character varying, numeric); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.czx_table_index_maint(p_run_type character varying, p_schema character varying, p_table character varying DEFAULT 'ALL'::character varying, currentjobid numeric DEFAULT NULL::numeric) RETURNS bigint
    LANGUAGE plpgsql SECURITY DEFINER
    SET search_path TO 'tm_cz', 'tm_lz', 'tm_wz', 'deapp', 'i2b2demodata', 'pg_temp'
    AS $$
/*
 *   Copyright 2012-2013 The Regents of the University of Colorado
 *
 *   Licensed under the Apache License, Version 2.0 (the "License")
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 */
declare

	--Audit variables
	newJobFlag		integer;
	databaseName 	VARCHAR(100);
	procedureName 	VARCHAR(100);
	jobID 			numeric(18,0);
	stepCt 			numeric(18,0);
	rowCt			numeric(18,0);
	errorNumber		character varying;
	errorMessage	character varying;
	rtnCd			integer;
	
	v_ct		int;
	v_index		record;
	v_sql		varchar(4000);
  
begin

	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;
	databaseName := 'tm_cz';
	procedureName := 'czx_table_index_maint';

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it

	IF(jobID IS NULL or jobID < 1)
	THEN
		newJobFlag := 1; -- True
		select tm_cz.czx_start_audit (procedureName, databaseName) into jobID;
	END IF;

	stepCt := 0;
	stepCt := stepCt + 1;
	select tm_cz.czx_write_audit(jobId,databaseName,procedureName,'Starting ' || procedureName,0,stepCt,'Done') into rtnCd;
	stepCt := stepCt + 1;
	select tm_cz.czx_write_audit(jobId,databaseName,procedureName,' run_type: ' || p_run_type || ' schema: ' || p_schema ||  ' table: ' || p_table,0,stepCt,'Done') into rtnCd;


	if p_run_type not in ('DROP','ADD','SAVE') then
		stepCt := stepCt + 1;
		select tm_cz.czx_write_audit(jobId,databaseName,procedureName,'Invalid run_type',0,stepCt,'Done') into rtnCd;
		select tm_cz.czx_error_handler (jobID, procedureName, '-1', 'Application raised error') into rtnCd;
		select tm_cz.czx_end_audit (jobID, 'FAIL') into rtnCd;
		return 16;
	end if;
	
	if p_schema is null or p_schema = '' then
		stepCt := stepCt + 1;
		select tm_cz.czx_write_audit(jobId,databaseName,procedureName,'Schema name missing',0,stepCt,'Done') into rtnCd;
		select tm_cz.czx_error_handler (jobID, procedureName, '-1', 'Application raised error') into rtnCd;
		select tm_cz.czx_end_audit (jobID, 'FAIL') into rtnCd;
		return 16;
	end if;
	
	select count(*) into v_ct
	from tm_cz.table_index
	where schema_name = p_schema
	  and case when p_run_type = 'ALL' then 1
	           else case when table_name = p_table then 1
			             else 0
						 end
			   end = 1;
	
	if v_ct = 0 then
		if p_run_type != 'SAVE' then
			if p_table = 'ALL' then
				stepCt := stepCt + 1;
				select tm_cz.czx_write_audit(jobId,databaseName,procedureName,'No saved indexes foound for schema',0,stepCt,'Done') into rtnCd;
			else
				stepCt := stepCt + 1;
				select tm_cz.czx_write_audit(jobId,databaseName,procedureName,'No saved indexes found for table',0,stepCt,'Done') into rtnCd;
			end if;
			select tm_cz.czx_error_handler (jobID, procedureName, '-1', 'Application raised error') into rtnCd;
			select tm_cz.czx_end_audit (jobID, 'FAIL') into rtnCd;
			return 16;
		end if;
	end if;
	
	--	SAVE indexes
	
	if p_run_type = 'SAVE' then
	
		--	check that indexes exist for supplied schema name
		
		select count(*) into v_ct
		from pg_class a
		inner join pg_index b
			  on  a.oid = b.indexrelid 
		inner join pg_class c
			  on  b.indrelid = c.oid
		inner join pg_attribute d
			  on  c.oid = d.attrelid 
			  and d.attnum = any(b.indkey)
		inner join pg_namespace n
			  on n.oid = a.relnamespace
		where a.relname not like 'pg_%'
		  and n.nspname = p_schema
		  and case when p_table = 'ALL' then 1
				   else case when p_table = c.relname then 1
							 else 0 end
					end = 1;
	
	if v_ct = 0 then
		if p_table = 'ALL' then
			stepCt := stepCt + 1;
			select tm_cz.czx_write_audit(jobId,databaseName,procedureName,'No indexes foound for schema',0,stepCt,'Done') into rtnCd;
		else
			stepCt := stepCt + 1;
			select tm_cz.czx_write_audit(jobId,databaseName,procedureName,'No indexes found for table',0,stepCt,'Done') into rtnCd;
		end if;
		select tm_cz.czx_error_handler (jobID, procedureName, '-1', 'Application raised error') into rtnCd;
		select tm_cz.czx_end_audit (jobID, 'FAIL') into rtnCd;
		return 16;
	end if;
	
		--	delete existing indexes for schema tables from cz.table_maint
		
		begin
		delete from tm_cz.table_index
		where schema_name = p_schema
		  and case when p_table = 'ALL' then 1
				   else case when p_table = table_name then 1
							 else 0 end
				   end = 1;
		get diagnostics rowCt := ROW_COUNT;
		exception 
		when others then
			errorNumber := SQLSTATE;
			errorMessage := SQLERRM;
			--Handle errors.
			select tm_cz.czx_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
			--End Proc
			select tm_cz.czx_end_audit (jobID, 'FAIL') into rtnCd;
			return 16;
		end;
		stepCt := stepCt + 1; 
		select tm_cz.czx_write_audit(jobId,databaseName,procedureName,'Deleted existing indexes for ' ||
				case when p_table = 'ALL' then p_schema else p_table end,rowCt,stepCt,'Done') into rtnCd;
		
		--	insert indexes for schema tables into cz.table_index
		
		begin
		insert into tm_cz.table_index
		(schema_name
		,table_name
		,index_name
		,index_sql
		)
		select distinct n.nspname as schema_name
			  ,c.relname as table_name
			  ,a.relname as index_name
			  ,pg_get_indexdef(b.indexrelid) as index_sql
		from pg_class a
		inner join pg_index b
			  on  a.oid = b.indexrelid 
		inner join pg_class c
			  on  b.indrelid = c.oid
		inner join pg_attribute d
			  on  c.oid = d.attrelid 
			  and d.attnum = any(b.indkey)
		inner join pg_namespace n
			  on n.oid = a.relnamespace
		where a.relname not like 'pg_%'
		  and n.nspname = p_schema
		  and case when p_table = 'ALL' then 1
				   else case when p_table = c.relname then 1
							 else 0 end
				   end = 1
		order by schema_name, table_name, index_name;
		get diagnostics rowCt := ROW_COUNT;
		exception 
		when others then
			errorNumber := SQLSTATE;
			errorMessage := SQLERRM;
			--Handle errors.
			select tm_cz.czx_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
			--End Proc
			select tm_cz.czx_end_audit (jobID, 'FAIL') into rtnCd;
			return 16;
		end;
		stepCt := stepCt + 1; 
		select tm_cz.czx_write_audit(jobId,databaseName,procedureName,'Inserted indexes for ' ||
				case when p_table = 'ALL' then p_schema else p_table end,rowCt,stepCt,'Done') into rtnCd;
	else 
		--	drop indexes for DROP or ADD
		
		for v_index in (select * from tm_cz.table_index 
						where schema_name = p_schema 
						  and case when p_table = 'ALL' then 1
								   else case when p_table = table_name then 1
											 else 0
											 end
								   end = 1)
		loop
			v_sql = 'drop index if exists ' || v_index.schema_name || '.' || v_index.index_name || ' cascade';
			begin
			execute v_sql;
			get diagnostics rowCt := ROW_COUNT;
			--exception 
			--when others then
			--	errorNumber := SQLSTATE;
			--	errorMessage := SQLERRM;
				--Handle errors.
			--	select tm_cz.czx_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
				--End Proc
			--	select tm_cz.czx_end_audit (jobID, 'FAIL') into rtnCd;
			--	return 16;
			end;
			stepCt := stepCt + 1;
			select tm_cz.czx_write_audit(jobId,databaseName,procedureName,'Dropped index ' || v_index.index_name || ' for ' ||
				case when p_table = 'ALL' then p_schema else p_table end,rowCt,stepCt,'Done') into rtnCd;
			
			if p_run_type = 'ADD' then
				--	add indexes for schema
				v_sql = v_index.index_sql;
				begin
				execute v_sql;
				get diagnostics rowCt := ROW_COUNT;
				exception 
				when others then
					stepCt := stepCt + 1;
					select tm_cz.czx_write_audit(jobId,databaseName,procedureName,v_index.index_sql || ': ' || sqlerrm,0,stepCt,'Done') into rtnCd;
					errorNumber := SQLSTATE;
					errorMessage := SQLERRM;
					--Handle errors.
					select tm_cz.czx_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
					--End Proc
					select tm_cz.czx_end_audit (jobID, 'FAIL') into rtnCd;
					return 16;
				end;
				stepCt := stepCt + 1;
				select tm_cz.czx_write_audit(jobId,databaseName,procedureName, 'Added index ' || v_index.index_name || ' on ' || p_schema || '.' || v_index.table_name,0,stepCt,'Done') into rtnCd;
			end if;
		end loop;
	end if;

    ---Cleanup OVERALL JOB if this proc is being run standalone

	stepCt := stepCt + 1;
	select tm_cz.czx_write_audit(jobId,databaseName,procedureName,'End ' || procedureName,0,stepCt,'Done') into rtnCd;

	---Cleanup OVERALL JOB if this proc is being run standalone
	IF newJobFlag = 1
	THEN
		select tm_cz.czx_end_audit (jobID, 'SUCCESS') into rtnCd;
	END IF;

	return 1;
	

end
$$;


ALTER FUNCTION tm_cz.czx_table_index_maint(p_run_type character varying, p_schema character varying, p_table character varying, currentjobid numeric) OWNER TO tm_cz;

--
-- Name: czx_write_audit(numeric, character varying, character varying, character varying, numeric, numeric, character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.czx_write_audit(jobid numeric, databasename character varying, procedurename character varying, stepdesc character varying, recordsmanipulated numeric, stepnumber numeric, stepstatus character varying) RETURNS numeric
    LANGUAGE plpgsql SECURITY DEFINER
    SET search_path TO 'tm_cz', 'pg_temp'
    AS $$
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
DECLARE
	lastTime timestamp;
	currTime timestamp;
	elapsedSecs	numeric;
	rtnCd		numeric;
	
BEGIN

	select max(job_date)
    into lastTime
    from tm_cz.cz_job_audit
    where job_id = jobID;
	
	--	clock_timestamp() is the current system time
	
	select clock_timestamp() into currTime;

	elapsedSecs :=	coalesce(((DATE_PART('day', currTime - lastTime) * 24 + 
				   DATE_PART('hour', currTime - lastTime)) * 60 +
				   DATE_PART('minute', currTime - lastTime)) * 60 +
				   DATE_PART('second', currTime - lastTime),0);

	begin
	insert into tm_cz.cz_job_audit
	(job_id
	,database_name
 	,procedure_name
 	,step_desc
	,records_manipulated
	,step_number
	,step_status
    ,job_date
    ,time_elapsed_secs
	)
	values(
 		jobId,
		databaseName,
		procedureName,
		stepDesc,
		recordsManipulated,
		stepNumber,
		stepStatus,
		currTime,
		elapsedSecs);
	exception 
	when OTHERS then
		--raise notice 'proc failed state=%  errm=%', SQLSTATE, SQLERRM;
		select tm_cz.czx_write_error(jobId,'0'::character varying(10),SQLERRM::character varying(1000),SQLSTATE::character varying(1000),SQLERRM::character varying(1000)) into rtnCd;
		return -16;
	end;
	
	raise notice 'step: %', stepDesc;
	
	return 1;
END;
$$;


ALTER FUNCTION tm_cz.czx_write_audit(jobid numeric, databasename character varying, procedurename character varying, stepdesc character varying, recordsmanipulated numeric, stepnumber numeric, stepstatus character varying) OWNER TO tm_cz;

--
-- Name: czx_write_error(numeric, character varying, character varying, character varying, character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.czx_write_error(jobid numeric, errornumber character varying, errormessage character varying, errorstack character varying, errorbacktrace character varying) RETURNS numeric
    LANGUAGE plpgsql SECURITY DEFINER
    SET search_path TO 'tm_cz', 'pg_temp'
    AS $$
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/

BEGIN

	begin
	insert into tm_cz.cz_job_error(
		job_id,
		error_number,
		error_message,
		error_stack,
		error_backtrace,
		seq_id)
	select
		jobID,
		errorNumber,
		errorMessage,
		errorStack,
		errorBackTrace,
		max(seq_id) 
  from tm_cz.cz_job_audit 
  where job_id=jobID;
  
  end;
  
  return 1;
 
  exception 
	when OTHERS then
		raise notice 'proc failed state=%  errm=%', SQLSTATE, SQLERRM;
		return -16; 

END;
$$;


ALTER FUNCTION tm_cz.czx_write_error(jobid numeric, errornumber character varying, errormessage character varying, errorstack character varying, errorbacktrace character varying) OWNER TO tm_cz;

--
-- Name: czx_write_info(numeric, numeric, numeric, character varying, character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.czx_write_info(jobid numeric, messageid numeric, messageline numeric, messageprocedure character varying, infomessage character varying) RETURNS numeric
    LANGUAGE plpgsql SECURITY DEFINER
    SET search_path TO 'tm_cz', 'pg_temp'
    AS $$
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/

declare

	rtnCd	numeric;
	
BEGIN

	begin
	insert into tm_cz.cz_job_message
    (
      job_id,
      message_id,
      message_line,
      message_procedure,
      info_message,
      seq_id
    )
	select
      jobID,
      messageID,
      messageLine,
      messageProcedure,
      infoMessage,
      max(seq_id)
  from
    tm_cz.cz_job_audit
  where
    job_id = jobID;
  	exception 
	when OTHERS then
		--raise notice 'proc failed state=%  errm=%', SQLSTATE, SQLERRM;
		select tm_cz.cz_write_error(jobId,SQLSTATE,SQLERRM,null,null) into rtnCd;
		return -16;
	end;
	
	return 1;

END;
$$;


ALTER FUNCTION tm_cz.czx_write_info(jobid numeric, messageid numeric, messageline numeric, messageprocedure character varying, infomessage character varying) OWNER TO tm_cz;

--
-- Name: data_export(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.data_export() RETURNS void
    LANGUAGE plpgsql
    AS $$
DECLARE

--Iterate through a cursor of all patient IDs
--Dynamically build a sql statement
--Run the statement returning the results

cPatients CURSOR FOR
  SELECT distinct a.patient_num
    FROM observation_fact a
  join i2b2 b
    on a.concept_cd = b.c_basecode
  where c_fullname like '%BRC Depression Study%'
    and c_visualattributes not like '%H%'
  order by patient_num;

  dynamicSQL varchar(32767);
  dynamicSQL2 varchar(32767);


BEGIN
  dynamicSQL := 'select c_name ,c_fullname ';
  dynamicSQL2 := 'select c_name ,c_fullname ';

  FOR r_cPatients in cPatients Loop

    dynamicSQL  := dynamicSQL  || ',max(decode(patient_num,' || cast(r_cPatients.patient_num as varchar) || ',tval_char,null)) "' || cast(r_cPatients.patient_num as varchar) || '"';
    dynamicSQL2 := dynamicSQL2 || ',max(decode(patient_num,' || cast(r_cPatients.patient_num as varchar) || ',cast(nval_num as varchar(20)),null)) "' || cast(r_cPatients.patient_num as varchar) || '"';

  END LOOP;

  dynamicSQL := dynamicSQL || ' from observation_fact a join i2b2 b on a.concept_cd = b.c_basecode where c_fullname like ''%BRC Depression Study%'' and c_columndatatype = ''T'' and c_visualattributes not like ''%H%'' group by c_name, c_fullname';
  dynamicSQL2 := dynamicSQL2 || ' from observation_fact a join i2b2 b on a.concept_cd = b.c_basecode where c_fullname like ''%BRC Depression Study%'' and c_columndatatype = ''N'' and c_visualattributes not like ''%H%'' group by c_name, c_fullname order by c_fullname';

  EXECUTE(dynamicSQL || ' UNION ALL ' || dynamicsql2);

  RAISE NOTICE '%', dynamicSQL;-- || ' UNION ALL ' || dynamicsql2);
  RAISE NOTICE 'UNION ALL';
  RAISE NOTICE '%', dynamicsql2;
END;
 
$$;


ALTER FUNCTION tm_cz.data_export() OWNER TO tm_cz;

--
-- Name: drop_table(character varying, character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.drop_table(tabowner character varying, tabname character varying) RETURNS void
    LANGUAGE plpgsql
    AS $$
DECLARE

    temp integer:=0;
    drp_stmt varchar(200):=null;

    
BEGIN
      select
        count(*)
      into
        temp
      from
        all_tables
      where
        upper(TABLE_NAME) = upper(TabName)
      and
        upper(OWNER) = upper(TabOwner);

      if temp = 1 then
        drp_stmt := 'Drop Table ' || TabOwner || '.' || TabName;
        EXECUTE drp_stmt;
        commit;
      end if;

    EXCEPTION
      WHEN OTHERS THEN
      RAISE EXCEPTION 'An error was encountered - % -ERROR- %',SQLSTATE,SQLERRM;

END DROP_TABLE;

$$;


ALTER FUNCTION tm_cz.drop_table(tabowner character varying, tabname character varying) OWNER TO tm_cz;

--
-- Name: dropsyn(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.dropsyn() RETURNS void
    LANGUAGE plpgsql
    AS $$
DECLARE

 s_cur CURSOR FOR
 SELECT synonym_name
 FROM user_synonyms;

 RetVal  bigint;
 sqlstr  varchar(200);

BEGIN
  FOR s_rec IN s_cur LOOP
    sqlstr := 'DROP SYNONYM ' || s_rec.synonym_name;

    EXECUTE sqlstr;
    COMMIT;
  END LOOP;
END dropsyn;

$$;


ALTER FUNCTION tm_cz.dropsyn() OWNER TO tm_cz;

--
-- Name: find_user(character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.find_user(user_type character varying) RETURNS character varying
    LANGUAGE plpgsql
    AS $$
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
declare
	userName	character varying;
BEGIN

	if user_type = 'S' then
		select session_user into userName;
	else
		select current_user into userName;
	end if;

  
	return userName;
  

END;

$$;


ALTER FUNCTION tm_cz.find_user(user_type character varying) OWNER TO tm_cz;

--
-- Name: fract_error_dist(double precision); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.fract_error_dist(norminput double precision) RETURNS double precision
    LANGUAGE plpgsql IMMUTABLE
    AS $$
DECLARE
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/

  -- temp var
  t1 DOUBLE PRECISION;

  -- exponent input to next equation
  exponent_input DOUBLE PRECISION;

  ans DOUBLE PRECISION;

  fractError DOUBLE PRECISION;

BEGIN
  t1 := 1.0 / (1.0 + 0.5 * ABS(normInput));
  exponent_input := -normInput*normInput - 1.26551223 +
                           t1*(1.00002368 + t1*(0.37409196 + t1*(0.09678418 + t1*(-0.18628806 + t1*(0.27886807 + t1*(-1.13520398 + t1*(1.48851587 + t1*(-0.82215223 + t1*(0.17087277)))))))));

  -- Horner's method
  BEGIN
    ans := 1 - t1 * EXP(exponent_input);
  EXCEPTION
  WHEN numeric_value_out_of_range THEN --underflow or overflow
    IF exponent_input < 0 THEN
      ans := 1 - t1 * 0::double precision;
    ELSE
      ans := 1 - t1 * 'Infinity'::double precision;
    END IF;
  END;

  -- handle sign
  IF normInput>0 THEN fractError:= ans; ELSE fractError:= -ans; END IF;
  return fractError;
END
$$;


ALTER FUNCTION tm_cz.fract_error_dist(norminput double precision) OWNER TO tm_cz;

--
-- Name: i2b2_add_node(character varying, character varying, character varying, numeric); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_add_node(trialid character varying, path character varying, path_name character varying, currentjobid numeric) RETURNS integer
    LANGUAGE plpgsql SECURITY DEFINER
    SET search_path TO 'tm_cz', 'i2b2metadata', 'i2b2demodata', 'pg_temp'
    AS $$
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
Declare

  root_node		varchar(2000);
  root_level	integer;
  rtnCd			integer;
  
	--Audit variables
	newJobFlag		integer;
	databaseName 	VARCHAR(100);
	procedureName 	VARCHAR(100);
	jobID 			numeric(18,0);
	stepCt 			numeric(18,0);
	rowCt			numeric(18,0);
	errorNumber		character varying;
	errorMessage	character varying;
  
BEGIN
    
	stepCt := 0;
	
	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;

	databaseName := 'TM_CZ';
	procedureName := 'I2B2_ADD_NODE';
	
	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it
	IF(jobID IS NULL or jobID < 1)
	THEN
		newJobFlag := 1; -- True
		select tm_cz.cz_start_audit (procedureName, databaseName) into jobId;
	END IF;
  
	select tm_cz.parse_nth_value(path, 2, '\') into root_node;
	
	select c_hlevel into root_level
	from i2b2metadata.table_access
	where c_name = root_node;
  
	if path = ''  or path = '%' or path_name = ''
	then 
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Path or Path name missing, no action taken',0,stepCt,'Done') into rtnCd;
		return 1;
	end if;
	

	--Delete existing data.

	DELETE FROM i2b2demodata.OBSERVATION_FACT 
	WHERE concept_cd IN (SELECT C_BASECODE FROM i2b2metadata.I2B2 WHERE C_FULLNAME = PATH);
	get diagnostics rowCt := ROW_COUNT;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Deleted any concepts for path from I2B2DEMODATA observation_fact',rowCt,stepCt,'Done') into rtnCd;

	--CONCEPT DIMENSION
	DELETE FROM i2b2demodata.CONCEPT_DIMENSION
	WHERE CONCEPT_PATH = path;
	get diagnostics rowCt := ROW_COUNT;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Deleted any concepts for path from I2B2DEMODATA concept_dimension',rowCt,stepCt,'Done') into rtnCd;
    
	--I2B2
	DELETE FROM i2b2metadata.i2b2
	WHERE C_FULLNAME = PATH;
	get diagnostics rowCt := ROW_COUNT;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Deleted path from I2B2METADATA i2b2',rowCt,stepCt,'Done') into rtnCd;

	--	Insert new node
	
	--CONCEPT DIMENSION
	INSERT INTO i2b2demodata.CONCEPT_DIMENSION
	(CONCEPT_CD, CONCEPT_PATH, NAME_CHAR,  UPDATE_DATE,  DOWNLOAD_DATE, IMPORT_DATE, SOURCESYSTEM_CD)
	VALUES
	(cast(nextval('i2b2demodata.concept_id') as varchar),
	path,
	path_name,
	current_timestamp,
	current_timestamp,
	current_timestamp,
	TrialID
	);
	get diagnostics rowCt := ROW_COUNT;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Inserted concept for path into I2B2DEMODATA concept_dimension',rowCt,stepCt,'Done') into rtnCD;

	--I2B2
	INSERT INTO i2b2metadata.I2B2
	(c_hlevel, C_FULLNAME, C_NAME, C_VISUALATTRIBUTES, c_synonym_cd, C_FACTTABLECOLUMN, C_TABLENAME, C_COLUMNNAME,
	C_DIMCODE, C_TOOLTIP, UPDATE_DATE, DOWNLOAD_DATE, IMPORT_DATE, SOURCESYSTEM_CD, c_basecode, C_OPERATOR, c_columndatatype, c_comment,
	m_applied_path)
	SELECT 
	(length(concept_path) - coalesce(length(replace(concept_path, '\','')),0)) / length('\') - 2 + root_level,
	CONCEPT_PATH,
	NAME_CHAR,
	'FA',
	'N',
	'CONCEPT_CD',
	'CONCEPT_DIMENSION',
	'CONCEPT_PATH',
	CONCEPT_PATH,
	CONCEPT_PATH,
	current_timestamp,
	current_timestamp,
	current_timestamp,
	SOURCESYSTEM_CD,
	CONCEPT_CD,
	'LIKE',
	'T',
	case when TrialID is null then null else 'trial:' || TrialID end,
	'@'
	FROM i2b2demodata.CONCEPT_DIMENSION
	WHERE 
	CONCEPT_PATH = path;
	get diagnostics rowCt := ROW_COUNT;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Inserted path into I2B2METADATA i2b2',rowCt,stepCt,'Done') into rtnCd;
		
      ---Cleanup OVERALL JOB if this proc is being run standalone
	IF newJobFlag = 1
	THEN
		select tm_cz.cz_end_audit (jobID, 'SUCCESS') into rtnCD;
	END IF;

	return 1;
	
	EXCEPTION
	WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;

  
END;

$$;


ALTER FUNCTION tm_cz.i2b2_add_node(trialid character varying, path character varying, path_name character varying, currentjobid numeric) OWNER TO tm_cz;

--
-- Name: i2b2_add_root_node(character varying, numeric); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_add_root_node(root_node character varying, currentjobid numeric) RETURNS integer
    LANGUAGE plpgsql SECURITY DEFINER
    SET search_path TO 'tm_cz', 'i2b2metadata', 'pg_temp'
    AS $$
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
DECLARE	
	--Audit variables
	newJobFlag		integer;
	databaseName 	VARCHAR(100);
	procedureName 	VARCHAR(100);
	jobID 			numeric(18,0);
	stepCt 			numeric(18,0);
	rowCt			numeric(18,0);

	rootNode		varchar(200);
	rootPath		varchar(200);
	errorNumber		character varying;
	errorMessage	character varying;
	
	rtnCd			integer;
	
Begin
	rootNode := root_node;
	rootPath := '\' || rootNode || '\';

    stepCt := 0;
	
	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;

	databaseName := 'TM_CZ';
	procedureName := 'I2B2_ADD_ROOT_NODE';
	

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it
	IF(jobID IS NULL or jobID < 1)
	THEN
		newJobFlag := 1; -- True
		select tm_cz.cz_start_audit (procedureName, databaseName) into jobId;
	END IF;
	
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Start ' || procedureName,0,stepCt,'Done') into rtnCd;
	
	begin
	
	insert into i2b2metadata.table_access
	(c_table_cd
	,c_table_name
	,c_protected_access
	,c_hlevel
	,c_fullname
	,c_name
	,c_synonym_cd
	,c_visualattributes
	,c_totalnum
	,c_basecode
	,c_metadataxml
	,c_facttablecolumn
	,c_dimtablename
	,c_columnname
	,c_columndatatype
	,c_operator
	,c_dimcode
	,c_comment
	,c_tooltip
	,c_entry_date
	,c_change_date
	,c_status_cd
	,valuetype_cd
	)
	select rootNode as c_table_cd
		  ,'i2b2' as c_table_name
		  ,'N' as protected_access
		  ,0 as c_hlevel
		  ,rootPath as c_fullname
		  ,rootNode as c_name
		  ,'N' as c_synonym_cd
		  ,'CA' as c_visualattributes
		  ,null as c_totalnum
		  ,null as c_basecode
		  ,null as c_metadataxml
		  ,'concept_cd' as c_facttablecolumn
		  ,'concept_dimension' as c_dimtablename
		  ,'concept_path' as c_columnname
		  ,'T' as c_columndatatype
		  ,'LIKE' as c_operator
		  ,rootPath as c_dimcode
		  ,null as c_comment
		  ,rootPath as c_tooltip
		  ,current_timestamp as c_entry_date
		  ,null as c_change_date
		  ,null as c_status_cd
		  ,null as valuetype_cd
	where not exists
		(select 1 from i2b2metadata.table_access x
		 where x.c_table_cd = rootNode);
	get diagnostics rowCt := ROW_COUNT;
	
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Insert to table_access',rowCt,stepCt,'Done') into rtnCd;	

	--	insert root_node into i2b2
	
	insert into i2b2metadata.i2b2
	(c_hlevel
	,c_fullname
	,c_name
	,c_synonym_cd
	,c_visualattributes
	,c_totalnum
	,c_basecode
	,c_metadataxml
	,c_facttablecolumn
	,c_tablename
	,c_columnname
	,c_columndatatype
	,c_operator
	,c_dimcode
	,c_comment
	,c_tooltip
	,m_applied_path
	,update_date
	,download_date
	,import_date
	,sourcesystem_cd
	,valuetype_cd
	,m_exclusion_cd
	,c_path
	,c_symbol
	--,i2b2_id
	)
	select 0 as c_hlevel
		  ,rootPath as c_fullname
		  ,rootNode as c_name
		  ,'N' as c_synonym_cd
		  ,'CA' as c_visualattributes
		  ,null as c_totalnum
		  ,null as c_basecode
		  ,null as c_metadataxml
		  ,'concept_cd' as c_facttablecolumn
		  ,'concept_dimension' as c_tablename
		  ,'concept_path' as c_columnname
		  ,'T' as c_columndatatype
		  ,'LIKE' as c_operator
		  ,rootPath as c_dimcode
		  ,null as c_comment
		  ,rootPath as c_tooltip
		  ,'@' as m_applied_path
		  ,current_timestamp as update_date
		  ,null as download_date
		  ,current_timestamp as import_date
		  ,null as sourcesystem_cd
		  ,null as valuetype_cd
		  ,null as m_exclusion_cd
		  ,null as c_path
		  ,null as c_symbol
		  --	add trigger on i2b2 insert
		  --,nextval('i2b2metadata.i2b2_id_seq')
		  --,I2B2_ID_SEQ.nextval as i2b2_id
	where not exists
		 (select 1 from i2b2metadata.i2b2 x
		  where x.c_name = rootNode);
	get diagnostics rowCt := ROW_COUNT;
		  
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Insert root_node ' || rootNode || ' to i2b2',rowCt,stepCt,'Done') into rtnCd;
    
	end;
			
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'End ' || procedureName,0,stepCt,'Done') into rtnCD;
	
	--Cleanup OVERALL JOB if this proc is being run standalone
	IF newJobFlag = 1
	THEN
		select tm_cz.cz_end_audit (jobID, 'SUCCESS') into rtnCd;
	END IF;

	return 1;
	
	EXCEPTION
	WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	
END;

$$;


ALTER FUNCTION tm_cz.i2b2_add_root_node(root_node character varying, currentjobid numeric) OWNER TO tm_cz;

--
-- Name: i2b2_add_snp_biomarker_nodes(character varying, character varying, numeric); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_add_snp_biomarker_nodes(trial_id character varying, ont_path character varying, currentjobid numeric DEFAULT '-1'::integer) RETURNS void
    LANGUAGE plpgsql
    AS $_$
DECLARE

	--	Adds SNP platform and sample type nodes into Biomarker Data ontology and adds rows into observation_fact for
	--	each subject/concept combination
	
	--	JEA@20110120	New
	--	JEA@@0111218	Remove hard-coded "Biomarker Data" node, use what's supplied in ont_path

	TrialID	varchar(100);
	ontPath		varchar(500);
  
	RootNode	varchar(300);
	pExists 	integer;
	platformTitle	varchar(200);
	tText		varchar(1000);
	ontLevel	integer;
	nodeName	varchar(200);
    
	--Audit variables
	newJobFlag integer(1);
	databaseName varchar(100);
	procedureName varchar(100);
	jobID numeric;
	stepCt integer;
	
	--	raise exception if platform not in de_gpl_info
	
	missing_GPL exception;
	
--	cursor to add platform-level nodes, need to be inserted before de_subject_sample_mapping

	addPlatform CURSOR FOR
	SELECT distinct REGEXP_REPLACE(ont_path || '\' || g.title || '\' ,
                  '(\\){2,}', '\') as path
		   ,g.title
	from de_subject_snp_dataset s
		,de_gpl_info g
	where s.trial_name = TrialId
	  and coalesce(s.platform_name,'GPL570') = g.platform
	  and upper(g.organism) = 'HOMO SAPIENS';
	
--	cursor to add sample-level nodes

	addSample CURSOR FOR
	SELECT distinct REGEXP_REPLACE(ont_path || '\' || g.title || '\' ||
                    s.sample_type || '\',	'(\\){2,}', '\') as sample_path
		   ,s.sample_type as sample_name
	from de_subject_snp_dataset s
		,de_gpl_info g
	where s.trial_name = TrialId
	  and coalesce(s.platform_name,'GPL570') = g.platform
	  and upper(g.organism) = 'HOMO SAPIENS'
	  and (s.sample_type IS NOT NULL AND s.sample_type::text <> '');
	  

BEGIN
	TrialID := upper(trial_id); 
	ontPath := ont_path;
  
	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;

	PERFORM sys_context('USERENV', 'CURRENT_SCHEMA') INTO databaseName ;
	procedureName := $$PLSQL_UNIT;

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it
	IF(coalesce(jobID::text, '') = '' or jobID < 1)
	THEN
		newJobFlag := 1; -- True
		cz_start_audit (procedureName, databaseName, jobID);
	END IF;
    	
	stepCt := 0;
	cz_write_audit(jobId,databaseName,procedureName,'Starting i2b2_add_snp_node',0,stepCt,'Done');
	stepCt := stepCt + 1;

	--	determine last node in ontPath
	
	PERFORM length(ontPath)-length(replace(ontPath,'\','')) into ontLevel ;
	PERFORM parse_nth_value(ontPath,ontLevel,'\') into nodeName ;
	
	--	add the high level \ node if it doesn't exist (first time loading data)
  
	select count(*)
	into pExists
	from i2b2
	where c_fullname = REGEXP_REPLACE(ont_path || '\','(\\){2,}', '\');
  
	if pExists = 0 then 
		i2b2_add_node(TrialId, REGEXP_REPLACE(ont_path || '\','(\\){2,}', '\'), nodeName, jobID);
        stepCt := stepCt + 1;
	    cz_write_audit(jobId,databaseName,procedureName,'Add node for ontPath',0,stepCt,'Done');
	end if;

    --	check if a node exists for the platform, if yes, then delete existing data, make sure all platforms in de_subject_snp_dataset have an
	--	entry in de_gpl_info, if not, raise exception
	
	select count(*) into pExists
	from de_subject_snp_dataset s
		,de_gpl_info g
	where s.trial_name = TrialId
	  and coalesce(s.platform_name,'GPL570') = g.platform(+)
	  and  'HOMO SAPIENS' = upper(g.organism(+))
	  and coalesce(g.platform::text, '') = '';
	  
	if pExists > 0
		then raise missing_GPL;
	end if;

	--	add SNP platform nodes
	
	for r_addPlatform in addPlatform Loop
	
		i2b2_delete_all_nodes(REGEXP_REPLACE(ont_path || '\','(\\){2,}', '\') || r_addPlatform.title || '\', jobID);
		stepCt := stepCt + 1;
		cz_write_audit(jobId,databaseName,procedureName,'Delete existing SNP Platform for trial in I2B2METADATA i2b2',0,stepCt,'Done');
		
		i2b2_add_node(TrialId, r_addPlatform.path, r_addPlatform.title, jobId);
		tText := 'Added Platform: ' || r_addPlatform.path || '  Name: ' || r_addPlatform.title;
		stepCt := stepCt + 1;
	    cz_write_audit(jobId,databaseName,procedureName,tText,SQL%ROWCOUNT,stepCt,'Done');
	end loop;

	cz_write_audit(jobId,databaseName,procedureName,'Added SNP Platform nodes',0,stepCt,'Done');
	stepCt := stepCt + 1;
	commit;
	       
	--	Insert the sample-level nodes
	
	for r_addSample in addSample Loop
	
		i2b2_add_node(TrialId, r_addSample.sample_path, r_addSample.sample_name, jobId);
		tText := 'Added Sample: ' || r_addSample.sample_path || '  Name: ' || r_addSample.sample_name;
	    cz_write_audit(jobId,databaseName,procedureName,tText,SQL%ROWCOUNT,stepCt,'Done');
		stepCt := stepCt + 1;

	end loop;		  
	
--	Insert records for patients into observation_fact

	insert into observation_fact
    (patient_num
	,concept_cd
	,modifier_cd
	,valtype_cd
	,tval_char
	,nval_num
	,sourcesystem_cd
	,import_date
	,valueflag_cd
	,provider_id
	,location_cd
	,units_cd
    )
    PERFORM p.patient_num
		  ,t.concept_cd
		  ,t.sourcesystem_cd
		  ,'T' -- Text data type
		  ,'E'  --Stands for Equals for Text Types
		  ,null	--	not numeric for Proteomics
		  ,t.sourcesystem_cd
		  ,LOCALTIMESTAMP
		  ,'@'
		  ,'@'
		  ,'@'
		  ,'' -- no units available
    from  concept_dimension t
		 ,de_subject_snp_dataset p
		 ,de_gpl_info g
    where p.trial_name =  TrialId
	  and coalesce(p.platform_name,'GPL570') = g.platform
	  and upper(g.organism) = 'HOMO SAPIENS'
	  and t.concept_path = REGEXP_REPLACE(ont_path || '\','(\\){2,}', '\') || g.title || '\' || p.sample_type || '\'
    group by p.patient_num
			,t.concept_cd
			,t.sourcesystem_cd;
	cz_write_audit(jobId,databaseName,procedureName,'Insert trial into I2B2DEMODATA observation_fact',SQL%ROWCOUNT,stepCt,'Done');
	stepCt := stepCt + 1;
    commit;
	
	--	update concept_cd in de_subject_snp_dataset
	
	update de_subject_snp_dataset d
	set concept_cd = (select t.concept_cd
					  from de_subject_snp_dataset p
						  ,de_gpl_info g
						  ,concept_dimension t
					  where d.subject_snp_dataset_id = p.subject_snp_dataset_id
						and coalesce(p.platform_name,'GPL570') = g.platform
						and upper(g.organism) = 'HOMO SAPIENS'
						and t.concept_path = REGEXP_REPLACE(ont_path || '\','(\\){2,}', '\') || g.title || '\' || p.sample_type || '\'
					  )
	where d.trial_name = TrialId;
	
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Update concept_cd in DEAPP de_subject_snp_dataset',SQL%ROWCOUNT,stepCt,'Done');
    commit;
    
--	Update visual attributes for leaf active (default is folder)

	update i2b2 a
    set c_visualattributes = 'LA'
    where 1 = (select count(*)
			   from i2b2 b
			   where b.c_fullname like (a.c_fullname || '%'))
      and a.c_fullname like REGEXP_REPLACE(ont_path || '\','(\\){2,}', '\') || '%';
    stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Update leaf active attribute for trial in I2B2METADATA i2b2',SQL%ROWCOUNT,stepCt,'Done');
    commit;
	
--	fill in tree

	--	get top level for study, this will be used for fill-in and create_concept_counts
	--	if this fails, check to make sure the trialId is not a sourcesystem_cd at an higher level than the study
	
	select b.c_fullname into nodeName
	from i2b2 b
	where b.c_hlevel =
		 (select min(x.c_hlevel) from i2b2 x
		  where b.sourcesystem_cd = x.sourcesystem_cd)
	  and ontPath like b.c_fullname || '%'
	  and b.sourcesystem_cd = TrialId;

	i2b2_fill_in_tree(TrialID,REGEXP_REPLACE(nodeName || '\','(\\){2,}', '\'), jobID);
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Fill in tree for Biomarker Data for trial',SQL%ROWCOUNT,stepCt,'Done');
  
  --Build concept Counts
  --Also marks any i2B2 records with no underlying data as Hidden, need to do at Biomarker level because there may be multiple platforms and patient count can vary
  
    i2b2_create_concept_counts(REGEXP_REPLACE(nodeName || '\','(\\){2,}', '\'),jobID );
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Create concept counts',0,stepCt,'Done');

  --Reload Security: Inserts one record for every I2B2 record into the security table

    i2b2_load_security_data(jobId);
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Load security data',0,stepCt,'Done');

	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'End i2b2_process_protein_data',0,stepCt,'Done');
		
    ---Cleanup OVERALL JOB if this proc is being run standalone
  IF newJobFlag = 1
  THEN
    cz_end_audit (jobID, 'SUCCESS');
  END IF;

  EXCEPTION

  WHEN missing_GPL  then
	--	put message in log
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'One or more GPL platforms in de_subject_snp_dataset is not in de_gpl_info',0,stepCt,'Done');

    --End Proc
    cz_end_audit (jobID, 'FAIL');

  WHEN OTHERS THEN
    --Handle errors.
    cz_error_handler (jobID, procedureName);
    --End Proc
    cz_end_audit (jobID, 'FAIL'); 

END;
 
$_$;


ALTER FUNCTION tm_cz.i2b2_add_snp_biomarker_nodes(trial_id character varying, ont_path character varying, currentjobid numeric) OWNER TO tm_cz;

--
-- Name: i2b2_backout_trial(character varying, character varying, numeric); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_backout_trial(trialid character varying, path_string character varying, currentjobid numeric) RETURNS integer
    LANGUAGE plpgsql SECURITY DEFINER
    SET search_path TO 'tm_cz', 'i2b2metadata', 'i2b2demodata', 'pg_temp'
    AS $$

/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
Declare

	rtnCd			integer;
	pExists			integer;
	sqlTxt			character varying;
	topNode			character varying;
	v_partition_id		text;
	secureObjId		bigint;

	--Audit variables
	newJobFlag		integer;
	databaseName		VARCHAR(100);
	procedureName		VARCHAR(100);
	jobId 			numeric(18,0);
	stepCt 			numeric(18,0);
	rowCt			numeric(18,0);
	errorNumber		character varying;
	errorMessage	        character varying;
	auditMessage	        character varying;

BEGIN

	stepCt := 0;
	
	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobId := currentjobId;

	databaseName := 'TM_CZ';
	procedureName := 'i2b2_backout_trial';
	
	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it
	IF(jobId IS NULL or jobId < 1)
	THEN
		newJobFlag := 1; -- True
		select tm_cz.cz_start_audit (procedureName, databaseName) into jobId;
	END IF;

	if (trialid is null OR length(trialid) = 0)
	then
		errorNumber := '';
		errorMessage := 'Invalid value for trialid argument';
		--Handle errors.
		select tm_cz.cz_error_handler (jobId, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobId, 'FAIL') into rtnCd;
		return -16;
	end if;

	-- trial id's are stored in upper case in the database
	trialid := upper(trialid);

	-- The second argument for this stored procedure (path_string) is deprecated.
	-- It's value is not independent of the value of the first argument (trialid) and
	-- therefore could be retrieved for the database. This prevents that errors are introduced
	-- in case inconsistent values are used for those arguments.
	-- For now, if a value is provided for path_string, it is checked against the information in the database
	-- and if there is no match, the procedure will be aborted.
	rowCt := 0;
	stepCt := stepCt + 1;
	if (path_string is not null AND length(path_string) > 0)
	then
		auditMessage := 'The use of the path_string argument for this stored procedure (i2b2_backout_trial) is deprecated';
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,auditMessage,rowCt,stepCt,'Done') into rtnCd;
	end if;	

	-- retrieve topNode for study with given trial id (trialid)
	begin
	select c_fullname from i2b2metadata.i2b2 where sourcesystem_cd = trialid and c_hlevel = 1 into topNode;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobId, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobId, 'FAIL') into rtnCd;
		return -16;
	end;

	-- check validity of topNode value
	rowCt := 0;
	stepCt := stepCt + 1;
	if (topNode is null OR length(topNode) = 0)
	then
        -- Either due to erroneous trailid or inconsistent database content
        -- In case of erroneous trialid, the database does not contain any data associated with this trialid (trying to remove does not harm)
        -- In case of inconsistencies in the database, we still might try to remove data associate with this trialid.
		auditMessage := 'Not able to retrieve top node associated with trial id: ' || trialid;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,auditMessage,rowCt,stepCt,'Done') into rtnCd;
	end if;

	-- check consistency between topNode and path_string
	rowCt := 0;
	stepCt := stepCt + 1;
	if (path_string is not null AND length(path_string) > 0 AND topNode is not null AND path_string != topNode)
	then
		errorNumber := '';
		errorMessage := 'Discrepancy between path_string argument value ('||path_string||') and value found in database ('||topNode||')';
		--Handle errors.
		select tm_cz.cz_error_handler (jobId, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobId, 'FAIL') into rtnCd;
		return -16;
	end if;

	-- delete all i2b2 nodes
	rowCt := 0;
	stepCt := stepCt + 1;
	if (topNode is null OR length(topNode) = 0)
	then
		auditMessage := 'Not able to retrieve top node associated with trial id: ' || trialid;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,auditMessage,rowCt,stepCt,'Done') into rtnCd;

		auditMessage := 'Start deleting all data for trial ' || trialid;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,auditMessage,rowCt,stepCt,'Done') into rtnCd;
	else
		auditMessage := 'Start deleting all data for trial ' || trialid || ' and topNode ' || topNode;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,auditMessage,rowCt,stepCt,'Done') into rtnCd;

		select tm_cz.i2b2_delete_all_nodes(topNode,jobId) into rtnCd;	
	end if;

	--	delete clinical data
	
	begin
	delete from tm_lz.lz_src_clinical_data
	where study_id = trialid;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobId, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobId, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete data for trial from lz_src_clinical_data',rowCt,stepCt,'Done') into rtnCd;

	--	delete observation_fact SECURITY data, do before patient_dimension delete
	
	begin
	delete from i2b2demodata.observation_fact f
	where f.concept_cd = 'SECURITY'
	  and f.patient_num in
	     (select distinct p.patient_num from i2b2demodata.patient_dimension p
		  where p.sourcesystem_cd like trialid || ':%');
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobId, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobId, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete SECURITY data for trial from I2B2DEMODATA observation_fact',rowCt,stepCt,'Done') into rtnCd;

	--	delete patient data
	
	begin
	delete from i2b2demodata.patient_dimension
	where sourcesystem_cd like trialid || ':%';
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobId, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobId, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete data for trial from I2B2DEMODATA patient_dimension',rowCt,stepCt,'Done') into rtnCd;

	begin
	delete from i2b2demodata.patient_trial
	where trial=  trialid;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobId, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobId, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete data for trial from I2B2DEMODATA patient_trial',rowCt,stepCt,'Done') into rtnCd;

--	delete gene expression data
	
	select count(*) into pExists
	from deapp.de_subject_sample_mapping
	where trial_name = TrialId
	  and platform = 'MRNA_AFFYMETRIX'
	  and trial_name = TrialId
	  and coalesce(omic_source_study,trial_name) = TrialId;

	if pExists > 0 then

		for v_partition_id in
			select distinct partition_id::text
			from deapp.de_subject_sample_mapping
				where trial_name = TrialId
				and platform = 'MRNA_AFFYMETRIX'
				and coalesce(omic_source_study,trial_name) = TrialId
		loop
			sqlTxt := 'drop table if exists deapp.de_subject_microarray_data_' || v_partition_id;
			execute sqlTxt;
			stepCt := stepCt + 1;
			select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Drop partition table for de_subject_microarray_data',rowCt,stepCt,'Done') into rtnCd;
		end loop;
		
		begin
		delete from deapp.de_subject_sample_mapping
		where trial_name = TrialID
		  and platform = 'MRNA_AFFYMETRIX';
		get diagnostics rowCt := ROW_COUNT;
		exception
		when others then
			errorNumber := SQLSTATE;
			errorMessage := SQLERRM;
			--Handle errors.
			select tm_cz.cz_error_handler (jobId, procedureName, errorNumber, errorMessage) into rtnCd;
			--End Proc
			select tm_cz.cz_end_audit (jobId, 'FAIL') into rtnCd;
			return -16;
		end;
		stepCt := stepCt + 1;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete data for trial from DEAPP de_subject_sample_mapping',rowCt,stepCt,'Done') into rtnCd;
		
	end if;
	
	--	delete acgh data
	
	select count(*) into pExists
	from deapp.de_subject_sample_mapping
	where trial_name = TrialId
	  and platform = 'ACGH'
	  and trial_name = TrialId
	  and coalesce(omic_source_study,trial_name) = TrialId;

	if pExists > 0 then

		for v_partition_id in
			select distinct partition_id::text
			from deapp.de_subject_sample_mapping
				where trial_name = TrialId
				and platform = 'ACGH'
				and coalesce(omic_source_study,trial_name) = TrialId
		loop
			sqlTxt := 'drop table if exists deapp.de_subject_acgh_data_' || v_partition_id;
			execute sqlTxt;
			stepCt := stepCt + 1;
			select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Drop partition table for de_subject_acgh_data',rowCt,stepCt,'Done') into rtnCd;
		end loop;
		
		begin
		delete from deapp.de_subject_sample_mapping
		where trial_name = TrialID
		  and platform = 'ACGH';
		get diagnostics rowCt := ROW_COUNT;
		exception
		when others then
			errorNumber := SQLSTATE;
			errorMessage := SQLERRM;
			--Handle errors.
			select tm_cz.cz_error_handler (jobId, procedureName, errorNumber, errorMessage) into rtnCd;
			--End Proc
			select tm_cz.cz_end_audit (jobId, 'FAIL') into rtnCd;
			return -16;
		end;
		stepCt := stepCt + 1;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete data for trial from DEAPP de_subject_sample_mapping',rowCt,stepCt,'Done') into rtnCd;
		
	end if;

	--	delete rnaseq data
	
	select count(*) into pExists
	from deapp.de_subject_sample_mapping
	where trial_name = TrialId
	  and platform = 'RNASEQ'
	  and trial_name = TrialId
	  and coalesce(omic_source_study,trial_name) = TrialId;

	if pExists > 0 then
		for v_partition_id in
			select distinct partition_id::text
			from deapp.de_subject_sample_mapping
				where trial_name = TrialId
				and platform = 'RNASEQ'
				and coalesce(omic_source_study,trial_name) = TrialId
		loop
			sqlTxt := 'drop table if exists deapp.de_subject_rnaseq_data_' || v_partition_id;
			execute sqlTxt;
			stepCt := stepCt + 1;
			select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Drop partition table for de_subject_rnaseq_data',rowCt,stepCt,'Done') into rtnCd;
		end loop;
		
		begin
		delete from deapp.de_subject_sample_mapping
		where trial_name = TrialID
		  and platform = 'RNASEQ';
		get diagnostics rowCt := ROW_COUNT;
		exception
		when others then
			errorNumber := SQLSTATE;
			errorMessage := SQLERRM;
			--Handle errors.
			select tm_cz.cz_error_handler (jobId, procedureName, errorNumber, errorMessage) into rtnCd;
			--End Proc
			select tm_cz.cz_end_audit (jobId, 'FAIL') into rtnCd;
			return -16;
		end;
		stepCt := stepCt + 1;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete data for trial from DEAPP de_subject_sample_mapping',rowCt,stepCt,'Done') into rtnCd;
		
	end if;

	-- delete from search_secure_object

	select count(*) into pExists
	from searchapp.search_secure_object
	where bio_data_unique_id = 'EXP:' || TrialId;

	if pExists > 0 then

		-- delete possible access right related to this secure object(s)
		delete from searchapp.search_auth_sec_object_access
		where secure_object_id in
		( select search_secure_object_id from searchapp.search_secure_object
		  where bio_data_unique_id = 'EXP:' || TrialId
		);

		-- delete the secure object(s)
		delete from searchapp.search_secure_object
		where bio_data_unique_id = 'EXP:' || TrialId;

		stepCt := stepCt + 1;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete trial from search_secure_object',rowCt,stepCt,'Done') into rtnCd;

	end if;

	-- Report potential erroneous value for trialid because topNode could not be found
	if (topNode is null OR length(topNode) = 0)
	then
		auditMessage := 'trialid argument possibly contained erroneous value ' || trialid;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,auditMessage,rowCt,stepCt,'Done') into rtnCd;
		select tm_cz.cz_end_audit (jobId, 'WARNING') into rtnCd;
		return -1;
	end if;

      ---Cleanup OVERALL JOB if this proc is being run standalone
	IF newJobFlag = 1
	THEN
		select tm_cz.cz_end_audit (jobId, 'SUCCESS') into rtnCD;
	END IF;

	return 1;
	
EXCEPTION
WHEN OTHERS THEN
	errorNumber := SQLSTATE;
	errorMessage := SQLERRM;
	--Handle errors.
	select tm_cz.cz_error_handler (jobId, procedureName, errorNumber, errorMessage) into rtnCd;
	--End Proc
	select tm_cz.cz_end_audit (jobId, 'FAIL') into rtnCd;
	return -16;
END;

$$;


ALTER FUNCTION tm_cz.i2b2_backout_trial(trialid character varying, path_string character varying, currentjobid numeric) OWNER TO tm_cz;

--
-- Name: i2b2_bulk_add_search_term(bigint); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_bulk_add_search_term(currentjobid bigint DEFAULT NULL::bigint) RETURNS void
    LANGUAGE plpgsql
    AS $_$
DECLARE

/*************************************************************************
* Copyright 2008-2012 Janssen Research null, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/

	--Audit variables
	newJobFlag integer(1);
	databaseName varchar(100);
	procedureName varchar(100);
	jobID bigint;
	stepCt bigint;

	v_keyword_term	varchar(500);
  v_display_category varchar(500);
  v_data_category varchar(500);
  v_prefix varchar(20);
  v_unique_ID varchar(500);
  v_source_cd varchar(500);
  v_parent_term varchar(500);
  v_bio_data_id bigint;
  
	sqlText			varchar(2000);
	Parent_Id 		integer;
	new_Term_Id 	integer;
	keyword_id 		integer;
	Lcount 			integer; 
	Ncount 			integer;
  
	--v_category_display	varchar2(200);

  	type keyword_rec  is record
	(keywordName		varchar(500)),
   dataCategory varchar(500),
   UIDPrefix varchar(20),
   uniqueID varchar(500),
   sourceCd varchar(500),
   parentTerm varchar(500),
   bio_data_id bigint
	);

	type keyword_table is table of keyword_rec; 
	keyword_array keyword_table;
	
  

BEGIN

	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;

	PERFORM sys_context('USERENV', 'CURRENT_SCHEMA') INTO databaseName ;
	procedureName := $$PLSQL_UNIT;

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it
	IF(coalesce(jobID::text, '') = '' or jobID < 1)
	THEN
		newJobFlag := 1; -- True
		cz_start_audit (procedureName, databaseName, jobID);
	END IF;
    	
	--stepCt := 0;
  
	--cz_write_audit(jobId,databaseName,procedureName,'Start Procedure',SQL%ROWCOUNT,stepCt,'Done');
	--Stepct := Stepct + 1;	
  
	  select keyword, display_category, data_category, UID_prefix, unique_id, source_cd, parent_term, bio_data_id
    bulk collect into keyword_array
  	from tm_lz.lt_src_search_keyword
    where (keyword IS NOT NULL AND keyword::text <> '');

 
	for i in keyword_array.first .. keyword_array.last
	loop
		v_keyword_term := keyword_array(i).keywordName;
    v_display_category := keyword_array(i).displayCategory;
    v_data_category := keyword_array(i).dataCategory;
    v_prefix := keyword_array(i).UIDPrefix;
    v_unique_ID := keyword_array(i).uniqueID;
    v_source_cd := keyword_array(i).sourceCd;
    v_parent_term := keyword_array(i).parentTerm;
    v_bio_data_id := keyword_array(i).bio_data_id;
    
    --dbms_output.put_line('keyword: ' || v_keyword_term);


		if (coalesce(v_display_category::text, '') = '') then
			v_display_category := v_data_category;
    end if;


    if (coalesce(v_unique_ID::text, '') = '') then
      if ((v_prefix IS NOT NULL AND v_prefix::text <> '')) then
        v_unique_ID := v_prefix || ':' || v_data_category || ':' || v_keyword_term;
      end if;
    end if;
   

		-- Insert taxonomy term into searchapp.search_keyword
		-- (searches Search_Keyword with the parent term to find the category to use)
		Insert Into Searchapp.Search_Keyword 
		(Data_Category
		,Keyword
		,Unique_Id
		,Source_Code
		,Display_Data_Category
    ,Bio_data_id)
		PERFORM v_data_category
			  ,v_keyword_term
			  ,v_unique_ID
			  ,v_source_cd
			  ,v_display_category
        ,v_bio_data_id
		
		where not exists
			(select 1 from searchapp.search_keyword x
			 where upper(x.data_category) = upper(v_data_category)
			   and upper(x.keyword) = upper(v_keyword_term)
         and upper(x.bio_data_id) = upper(v_bio_data_id));
		Cz_Write_Audit(Jobid,Databasename,Procedurename,v_keyword_term || ' added to Searchapp.Search_Keyword',Sql%Rowcount,Stepct,'Done');
		Stepct := Stepct + 1;	
		commit;
  
		-- Get the ID of the new term in Search_Keyword
		Select Search_Keyword_Id  Into keyword_Id 
		From  Searchapp.Search_Keyword 
		Where Upper(Keyword) = Upper(v_keyword_term)
		and upper(data_category) = upper(v_data_category)
    and upper(bio_data_id) = upper(v_bio_data_id);
		Cz_Write_Audit(Jobid,Databasename,Procedurename,'New search keyword ID stored in Keyword_Id',Sql%Rowcount,Stepct,'Done');
		Stepct := Stepct + 1;	
		
		-- Insert the new term into Searchapp.Search_v_keyword_term 
		insert into searchapp.search_keyword_term 
		(keyword_term
		,search_keyword_id
		,rank
		,term_length)
		PERFORM UPPER(v_keyword_term)
			  ,Keyword_Id
			  ,1
			  ,Length(v_keyword_term) 
		
		where not exists
			(select 1 from searchapp.search_keyword_term x
			 where upper(x.keyword_term) = upper(v_keyword_term)
			   and x.search_keyword_id = Keyword_Id);
		Cz_Write_Audit(Jobid,Databasename,Procedurename,'Term added to Searchapp.Search_v_keyword_term',Sql%Rowcount,Stepct,'Done');
		Stepct := Stepct + 1;	
		commit;

	end loop;

	Cz_Write_Audit(Jobid,Databasename,Procedurename,'End '|| procedureName,0,Stepct,'Done');
	Stepct := Stepct + 1;  

 
     ---Cleanup OVERALL JOB if this proc is being run standalone    
  IF newJobFlag = 1
  THEN
    cz_end_audit (jobID, 'SUCCESS');
  END IF;
  
    EXCEPTION
  WHEN OTHERS THEN
    --Handle errors.
    cz_error_handler (jobID, procedureName);
    --End Proc
    cz_end_audit (jobID, 'FAIL');
  
END;
 
$_$;


ALTER FUNCTION tm_cz.i2b2_bulk_add_search_term(currentjobid bigint) OWNER TO tm_cz;

--
-- Name: i2b2_create_concept_counts(character varying, numeric); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_create_concept_counts(path character varying, currentjobid numeric DEFAULT '-1'::integer) RETURNS numeric
    LANGUAGE plpgsql SECURITY DEFINER
    SET search_path TO 'tm_cz', 'i2b2demodata', 'i2b2metadata', 'pg_temp'
    AS $$
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
Declare
 
	--Audit variables
	newJobFlag		integer;
	databaseName 	VARCHAR(100);
	procedureName 	VARCHAR(100);
	jobID 			numeric(18,0);
	stepCt 			numeric(18,0);
	rowCt			numeric(18,0);
	errorNumber		character varying;
	errorMessage	character varying;
	rtnCd			numeric;
  
BEGIN
     
	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;

	databaseName := 'TM_CZ';
	procedureName := 'I2B2_CREATE_CONCEPT_COUNTS';

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it
	IF(jobID IS NULL or jobID < 1)
	THEN
		newJobFlag := 1; -- True
		select tm_cz.cz_start_audit (procedureName, databaseName) into jobID;
	END IF;
    	
	stepCt := 0;
  
	begin
	delete from i2b2demodata.concept_counts
	where concept_path like path || '%' escape '`';
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete counts for trial from I2B2DEMODATA concept_counts',rowCt,stepCt,'Done') into rtnCd;

	--	Join each node (folder or leaf) in the path to it's leaf in the work table to count patient numbers

	begin
	insert into i2b2demodata.concept_counts
	(concept_path
	,parent_concept_path
	,patient_count
	)
	select fa.c_fullname
		  ,ltrim(SUBSTR(fa.c_fullname, 1,instr(fa.c_fullname, '\',-1,2)))
		  ,count(distinct tpm.patient_num)
	from i2b2metadata.i2b2 fa
	    ,i2b2metadata.i2b2 la
		,i2b2demodata.observation_fact tpm
		,i2b2demodata.patient_dimension p
	where fa.c_fullname like path || '%' escape '`'
	  and substr(fa.c_visualattributes,2,1) != 'H'
	  and la.c_fullname like fa.c_fullname || '%' escape '`'
	  and la.c_visualattributes like 'L%'
	  and tpm.patient_num = p.patient_num
	  and la.c_basecode = tpm.concept_cd   -- outer join in oracle ???
	group by fa.c_fullname
			,ltrim(SUBSTR(fa.c_fullname, 1,instr(fa.c_fullname, '\',-1,2)));
	get diagnostics rowCt := ROW_COUNT;		
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Insert counts for trial into I2B2DEMODATA concept_counts',rowCt,stepCt,'Done') into rtnCd;
	
	--SET ANY NODE WITH MISSING OR ZERO COUNTS TO HIDDEN

	begin
	update i2b2metadata.i2b2
	set c_visualattributes = substr(c_visualattributes,1,1) || 'H' || substr(c_visualattributes,3,1)
	where c_fullname like path || '%' escape '`'
	  and (not exists
			 (select 1 from concept_counts nc
				  where c_fullname = nc.concept_path)
				 or
			 exists
				 (select 1 from concept_counts zc
				  where c_fullname = zc.concept_path
					and zc.patient_count = 0)
			  )
		and c_name != 'SECURITY';
	get diagnostics rowCt := ROW_COUNT;	
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Nodes hidden with missing/zero counts for trial into I2B2DEMODATA concept_counts',rowCt,stepCt,'Done') into rtnCd;
		
	---Cleanup OVERALL JOB if this proc is being run standalone
	IF newJobFlag = 1
	THEN
		select tm_cz.cz_end_audit (jobID, 'SUCCESS') into rtnCd;
	END IF;

	return 1;

END;

$$;


ALTER FUNCTION tm_cz.i2b2_create_concept_counts(path character varying, currentjobid numeric) OWNER TO tm_cz;

--
-- Name: i2b2_create_external_tables(character varying, character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_create_external_tables(tpmextfn character varying, catgextfn character varying) RETURNS void
    LANGUAGE plpgsql
    AS $$
DECLARE


sqltxt varchar(5000);


BEGIN

--  recreate CATEGORY_EXTRNL tabls with CATGExtFN parameter (filename in external file system)

sqltxt:='drop table i2b2_lz.category_extrnl';

  EXECUTE sqltxt;

sqltxt:='CREATE or REPLACE  TABLE "I2B2_LZ"."CATEGORY_EXTRNL"
   ( "STUDY_ID" varchar(100 BYTE),
	"CATEGORY_CD" varchar(100 BYTE),
	"CATEGORY_PATH" varchar(250 BYTE)
   )
   ORGANIZATION EXTERNAL
    ( TYPE ORACLE_LOADER
      DEFAULT DIRECTORY "BIOMART_LZ"
      ACCESS PARAMETERS
      ( records delimited by newline nologfile skip 1
        fields terminated by 0X"09"
        MISSING FIELD VALUES ARE NULL
            )
      LOCATION
       ( ' || '''' || CATGExtFn || '''' || '))';

   EXECUTE sqltxt;

--  recreate TIME_POINT_MEASUREMENT_EXTRNL tabls with TPMExtFN parameter (filename in external file system)

sqltxt:='drop table i2b2_lz.time_point_measurement_extrnl';

  EXECUTE sqltxt;

sqltxt:='    CREATE or REPLACE  TABLE "I2B2_LZ"."TIME_POINT_MEASUREMENT_EXTRNL"
   ("STUDY_ID" varchar(25 BYTE),
	"USUBJID" varchar(50 BYTE),
	"SITE_ID" varchar(10 BYTE),
	"SUBJECT_ID" varchar(10 BYTE),
	"VISIT_NAME" varchar(100 BYTE),
	"DATASET_NAME" varchar(500 BYTE),
	"SAMPLE_TYPE" varchar(100 BYTE),
	"DATA_LABEL" varchar(500 BYTE),
	"DATA_VALUE" varchar(500 BYTE),
	"CATEGORY_CD" varchar(100 BYTE),
	"PERIOD" varchar(100 BYTE)
   )
   ORGANIZATION EXTERNAL
    ( TYPE ORACLE_LOADER
      DEFAULT DIRECTORY "BIOMART_LZ"
      ACCESS PARAMETERS
      ( records delimited by newline nologfile skip 1
        fields terminated by 0X"09"
        MISSING FIELD VALUES ARE NULL
            )
      LOCATION ( ' || '''' || TPMExtFn || '''' ||  ') )';

    EXECUTE sqltxt;

END;

$$;


ALTER FUNCTION tm_cz.i2b2_create_external_tables(tpmextfn character varying, catgextfn character varying) OWNER TO tm_cz;

--
-- Name: i2b2_create_security_for_trial(character varying, character varying, numeric); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_create_security_for_trial(trial_id character varying, secured_study character varying DEFAULT 'N'::character varying, currentjobid numeric DEFAULT '-1'::integer) RETURNS numeric
    LANGUAGE plpgsql SECURITY DEFINER
    SET search_path TO 'tm_cz', 'i2b2demodata', 'i2b2metadata', 'searchapp', 'biomart', 'pg_temp'
    AS $$
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
Declare
 
	--Audit variables
	newJobFlag		integer;
	databaseName 	VARCHAR(100);
	procedureName 	VARCHAR(100);
	jobID 			numeric(18,0);
	stepCt 			numeric(18,0);
	rowCt			numeric(18,0);
	errorNumber		character varying;
	errorMessage	character varying;
	rtnCd			numeric;

	TrialID varchar(100);
	securedStudy varchar(5);
	pExists			integer;
	v_bio_experiment_id	numeric(18,0);

BEGIN
	TrialID := trial_id;
	securedStudy := secured_study;
	
	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;

	databaseName := 'TM_CZ';
	procedureName := 'I2B2_CREATE_SECURITY_FOR_TRIAL';
	
	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it
	IF(jobID IS NULL or jobID < 1)
	THEN
		newJobFlag := 1; -- True
		select tm_cz.cz_start_audit (procedureName, databaseName) into jobId;
	END IF;

	stepCt := 0;
  
	begin
	delete from i2b2demodata.observation_fact
	where sourcesystem_cd = TrialID
	  and concept_cd = 'SECURITY';
	get diagnostics rowCt := ROW_COUNT;	
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete security records for trial from I2B2DEMODATA observation_fact',rowCt,stepCt,'Done') into rtnCd;

	begin
	insert into i2b2demodata.observation_fact
    (encounter_num
        ,patient_num
	,concept_cd
        ,start_date
	,provider_id
	,modifier_cd
	,valtype_cd
	,tval_char
	,valueflag_cd
	,location_cd
	,update_date
	,download_date
	,import_date
	,sourcesystem_cd
	,instance_num
	)
	select patient_num
                  ,patient_num
		  ,'SECURITY'
                  ,current_timestamp
		  ,'@'
		  ,TrialId
		  ,'T'
		  ,case when securedStudy = 'N' then 'EXP:PUBLIC' else 'EXP:' || trialID end
		  ,'@'
		  ,'@'
		  ,current_timestamp
		  ,current_timestamp
		  ,current_timestamp
		  --,sourcesystem_cd
		  ,TrialId
		  ,1
	from i2b2demodata.patient_dimension
	where sourcesystem_cd like TrialID || ':%';
	get diagnostics rowCt := ROW_COUNT;	
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;	
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Insert security records for trial from I2B2DEMODATA observation_fact',rowCt,stepCt,'Done') into rtnCd;
	
	--	insert patients to patient_trial table
	
	begin
	delete from i2b2demodata.patient_trial
	where trial  = TrialID;
	get diagnostics rowCt := ROW_COUNT;	
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete data for trial from I2B2DEMODATA patient_trial',rowCt,stepCt,'Done') into rtnCd;

	begin
	insert into i2b2demodata.patient_trial
	(patient_num
	,trial
	,secure_obj_token
	)
	select patient_num, 
		   TrialID,
		   case when securedStudy = 'N' then 'EXP:PUBLIC' else 'EXP:' || trialID end
	from i2b2demodata.patient_dimension
	where sourcesystem_cd like TrialID || ':%';
	get diagnostics rowCt := ROW_COUNT;	
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Insert data for trial into I2B2DEMODATA patient_trial',rowCt,stepCt,'Done') into rtnCd;
	
	--	if secure study, then create bio_experiment record if needed and insert to search_secured_object
	
	select count(*) into pExists
	from searchapp.search_secure_object sso
	where bio_data_unique_id = 'EXP:' || TrialId;
	
	if pExists = 0 then
		--	if securedStudy = Y, add trial to searchapp.search_secured_object
		if securedStudy = 'Y' then
			select count(*) into pExists
			from biomart.bio_experiment
			where accession = TrialId;
			
			if pExists = 0 then
				begin
				insert into biomart.bio_experiment
				(title, accession, etl_id)
				select 'Metadata not available'
					  ,TrialId
					  ,'METADATA:' || TrialId;
				get diagnostics rowCt := ROW_COUNT;	
				exception
				when others then
					errorNumber := SQLSTATE;
					errorMessage := SQLERRM;
					--Handle errors.
					select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
					--End Proc
					select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
					return -16;	
				end;
				stepCt := stepCt + 1;
				select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Insert trial/study into biomart.bio_experiment',rowCt,stepCt,'Done') into rtnCd;
			end if;
			
			select bio_experiment_id into v_bio_experiment_id
			from biomart.bio_experiment
			where accession = TrialId;
			
			begin
			insert into searchapp.search_secure_object
			(bio_data_id
			,display_name
			,data_type
			,bio_data_unique_id
			)
			select v_bio_experiment_id
				  ,parse_nth_value(md.c_fullname,2,'\') || ' - ' || md.c_name as display_name
				  ,'BIO_CLINICAL_TRIAL' as data_type
				  ,'EXP:' || TrialId as bio_data_unique_id
			from i2b2metadata.i2b2 md
			where md.sourcesystem_cd = TrialId
			  and md.c_hlevel = 
				 (select min(x.c_hlevel) from i2b2metadata.i2b2 x
				  where x.sourcesystem_cd = TrialId)
			  and not exists
				 (select 1 from searchapp.search_secure_object so
				  where v_bio_experiment_id = so.bio_data_id);
			get diagnostics rowCt := ROW_COUNT;	
			exception
			when others then
				errorNumber := SQLSTATE;
				errorMessage := SQLERRM;
				--Handle errors.
				select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
				--End Proc
				select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
				return -16;	
			end;
			stepCt := stepCt + 1;
			select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Inserted trial/study into SEARCHAPP search_secure_object',rowCt,stepCt,'Done') into rtnCd;
		end if;
	else
		--	if securedStudy = N, delete entry from searchapp.search_secure_object
		if securedStudy = 'N' then
			begin
			delete from searchapp.search_secure_object
			where bio_data_unique_id = 'EXP:' || TrialId;
			get diagnostics rowCt := ROW_COUNT;	
			exception
			when others then
				errorNumber := SQLSTATE;
				errorMessage := SQLERRM;
				--Handle errors.
				select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
				--End Proc
				select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
				return -16;	
			end;
			stepCt := stepCt + 1;
			select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Deleted trial/study from SEARCHAPP search_secure_object',rowCt,stepCt,'Done') into rtnCd;
		end if;		
	end if;
     
 
    ---Cleanup OVERALL JOB if this proc is being run standalone
	IF newJobFlag = 1
	THEN
		select tm_cz.cz_end_audit (jobID, 'SUCCESS') into rtnCd;
	END IF;

	return 1;
	
	EXCEPTION
	WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;

END;

$$;


ALTER FUNCTION tm_cz.i2b2_create_security_for_trial(trial_id character varying, secured_study character varying, currentjobid numeric) OWNER TO tm_cz;

--
-- Name: i2b2_create_security_inc_trial(character varying, character varying, numeric); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_create_security_inc_trial(trial_id character varying, secured_study character varying DEFAULT 'N'::character varying, currentjobid numeric DEFAULT '-1'::integer) RETURNS numeric
    LANGUAGE plpgsql SECURITY DEFINER
    AS $$
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
Declare
 
	--Audit variables
	newJobFlag		integer;
	databaseName 	VARCHAR(100);
	procedureName 	VARCHAR(100);
	jobID 			numeric(18,0);
	stepCt 			numeric(18,0);
	rowCt			numeric(18,0);
	errorNumber		character varying;
	errorMessage	character varying;
	rtnCd			numeric;

	TrialID varchar(100);
	securedStudy varchar(5);
	pExists			integer;
	v_bio_experiment_id	numeric(18,0);

BEGIN
	TrialID := trial_id;
	securedStudy := secured_study;
	
	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;

	databaseName := 'TM_CZ';
	procedureName := 'I2B2_CREATE_SECURITY_INC_TRIAL';
	
	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it
	IF(jobID IS NULL or jobID < 1)
	THEN
		newJobFlag := 1; -- True
		select tm_cz.cz_start_audit (procedureName, databaseName) into jobId;
	END IF;

	stepCt := 0;
  
    -- inc-change keep the SECURITY rows in observation_fact
/*
	begin
	delete from i2b2demodata.observation_fact
	where sourcesystem_cd = TrialID
	  and concept_cd = 'SECURITY';
	get diagnostics rowCt := ROW_COUNT;	
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete security records for trial from I2B2DEMODATA observation_fact',rowCt,stepCt,'Done') into rtnCd;
*/

	begin
	insert into i2b2demodata.observation_fact
    (encounter_num
        ,patient_num
	,concept_cd
        ,start_date
	,provider_id
	,modifier_cd
	,valtype_cd
	,tval_char
	,valueflag_cd
	,location_cd
	,update_date
	,download_date
	,import_date
	,sourcesystem_cd
	,instance_num
	)
	select patient_num
                  ,patient_num
		  ,'SECURITY'
                  ,current_timestamp
		  ,'@'
		  ,TrialId
		  ,'T'
		  ,case when securedStudy = 'N' then 'EXP:PUBLIC' else 'EXP:' || trialID end
		  ,'@'
		  ,'@'
		  ,current_timestamp
		  ,current_timestamp
		  ,current_timestamp
		  --,sourcesystem_cd
		  ,TrialId
		  ,1
	from i2b2demodata.patient_dimension
	where sourcesystem_cd like TrialID || ':%'                 -- inc-change keep SECURITY concepts
        and patient_num not in (select distinct patient_num from observation_fact where concept_cd='SECURITY'  )
        ;
	get diagnostics rowCt := ROW_COUNT;	
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;	
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Insert security records for trial from I2B2DEMODATA observation_fact',rowCt,stepCt,'Done') into rtnCd;
	
	--	insert patients to patient_trial table
	-- inc-change do not delete patients
/*
	begin
	delete from i2b2demodata.patient_trial
	where trial  = TrialID;
	get diagnostics rowCt := ROW_COUNT;	
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete data for trial from I2B2DEMODATA patient_trial',rowCt,stepCt,'Done') into rtnCd;
*/

	begin
	insert into i2b2demodata.patient_trial
	(patient_num
	,trial
	,secure_obj_token
	)
	select patient_num, 
		   TrialID,
		   case when securedStudy = 'N' then 'EXP:PUBLIC' else 'EXP:' || trialID end
	from i2b2demodata.patient_dimension
	where sourcesystem_cd like TrialID || ':%'
            -- inc-change new patients only
	and patient_num not in (select patient_num from patient_trial) ;
	get diagnostics rowCt := ROW_COUNT;	
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Insert data for trial into I2B2DEMODATA patient_trial',rowCt,stepCt,'Done') into rtnCd;
	
	--	if secure study, then create bio_experiment record if needed and insert to search_secured_object
	
	select count(*) into pExists
	from searchapp.search_secure_object sso
	where bio_data_unique_id = 'EXP:' || TrialId;
	
	if pExists = 0 then
		--	if securedStudy = Y, add trial to searchapp.search_secured_object
		if securedStudy = 'Y' then
			select count(*) into pExists
			from biomart.bio_experiment
			where accession = TrialId;
			
			if pExists = 0 then
				begin
				insert into biomart.bio_experiment
				(title, accession, etl_id)
				select 'Metadata not available'
					  ,TrialId
					  ,'METADATA:' || TrialId;
				get diagnostics rowCt := ROW_COUNT;	
				exception
				when others then
					errorNumber := SQLSTATE;
					errorMessage := SQLERRM;
					--Handle errors.
					select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
					--End Proc
					select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
					return -16;	
				end;
				stepCt := stepCt + 1;
				select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Insert trial/study into biomart.bio_experiment',rowCt,stepCt,'Done') into rtnCd;
			end if;
			
			select bio_experiment_id into v_bio_experiment_id
			from biomart.bio_experiment
			where accession = TrialId;
			
			begin
			insert into searchapp.search_secure_object
			(bio_data_id
			,display_name
			,data_type
			,bio_data_unique_id
			)
			select v_bio_experiment_id
				  ,parse_nth_value(md.c_fullname,2,'\') || ' - ' || md.c_name as display_name
				  ,'BIO_CLINICAL_TRIAL' as data_type
				  ,'EXP:' || TrialId as bio_data_unique_id
			from i2b2metadata.i2b2 md
			where md.sourcesystem_cd = TrialId
			  and md.c_hlevel = 
				 (select min(x.c_hlevel) from i2b2metadata.i2b2 x
				  where x.sourcesystem_cd = TrialId)
			  and not exists
				 (select 1 from searchapp.search_secure_object so
				  where v_bio_experiment_id = so.bio_data_id);
			get diagnostics rowCt := ROW_COUNT;	
			exception
			when others then
				errorNumber := SQLSTATE;
				errorMessage := SQLERRM;
				--Handle errors.
				select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
				--End Proc
				select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
				return -16;	
			end;
			stepCt := stepCt + 1;
			select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Inserted trial/study into SEARCHAPP search_secure_object',rowCt,stepCt,'Done') into rtnCd;
		end if;
	else
		--	if securedStudy = N, delete entry from searchapp.search_secure_object
		if securedStudy = 'N' then
			begin
			-- inc-change retain searchapp.search_secure_object
			--delete from searchapp.search_secure_object
			--where bio_data_unique_id = 'EXP:' || TrialId;
			get diagnostics rowCt := ROW_COUNT;	
			exception
			when others then
				errorNumber := SQLSTATE;
				errorMessage := SQLERRM;
				--Handle errors.
				select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
				--End Proc
				select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
				return -16;	
			end;
			stepCt := stepCt + 1;
			select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Deleted trial/study from SEARCHAPP search_secure_object',rowCt,stepCt,'Done') into rtnCd;
		end if;		
	end if;
     
 
    ---Cleanup OVERALL JOB if this proc is being run standalone
	IF newJobFlag = 1
	THEN
		select tm_cz.cz_end_audit (jobID, 'SUCCESS') into rtnCd;
	END IF;

	return 1;
	
	EXCEPTION
	WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;

END;

$$;


ALTER FUNCTION tm_cz.i2b2_create_security_inc_trial(trial_id character varying, secured_study character varying, currentjobid numeric) OWNER TO tm_cz;

--
-- Name: i2b2_create_stage_tables(bigint); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_create_stage_tables(bigint) RETURNS integer
    LANGUAGE plpgsql IMMUTABLE SECURITY DEFINER
    AS $$
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
Declare

	--Audit variables
	newJobFlag 	int4;
	databaseName 	varchar(100);
	procedureName varchar(100);
	jobID 		numeric(18,0);
	stepCt 		numeric(18,0);
	rowCount	numeric(18,0);
	rtnCd integer;
	--	Define the abstract result set record
	
	r_stage_table		record;
	r_stage_column		record;
	
	--	Variables

	tText 			varchar(2000);
	pExists			int4;
	release_table	varchar(50);
	v_sqlerrm		varchar(1000);
	
	
	BEGIN	
	
	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := -1;

	databaseName := 'TM_CZ';
	procedureName := 'I2B2_CREATE_STAGE_TABLES';

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it
	IF(jobID IS NULL or jobID < 1)
	THEN
		newJobFlag := 1; -- True
		jobId := tm_cz.czx_start_audit (procedureName, databaseName);
	END IF;
    	
	stepCt := 0;	
	select tm_cz.czx_write_audit(jobId,databaseName,procedureName,'Starting ' || procedureName,0,stepCt,'Done')into rtnCd;
	
	for r_stage_table in
		select upper(table_owner) as table_owner
			  ,upper(table_name) as table_name
			  ,upper(study_specific) as study_specific
			  ,upper(stage_table_name) as stage_table_name
		from tm_cz.migrate_tables
	loop
		release_table := r_stage_table.stage_table_name;

		select count(*) into pExists
		from _v_table
		where schema = 'TM_STAGE'
		  and tablename = release_table;	
	  
		if pExists > 0 then
			tText := 'drop table tm_stage.' || release_table;
			execute immediate tText;
		end if;

		tText := 'create table tm_stage.' || release_table || ' (';
		
		for r_stage_column in
			select attname as column_name
				  ,format_type
			from _v_relation_column 
			where name=upper(r_stage_table.table_name)
			order by attnum asc
		loop
			tText := tText || r_stage_column.column_name || ' ' || r_stage_column.format_type || ',';	
		end loop;
		
		if r_stage_table.study_specific = 'Y' then
			tText := tText || 'release_study varchar(200))';
		else
			tText := trim(trailing ',' from tText) || ')';
		end if;
		execute immediate tText;
		stepCt := stepCt + 1;
		select tm_cz.czx_write_audit(jobId,databaseName,procedureName,'Created '|| release_table,0,stepCt,'Done')into rtnCd;
			
	end loop;
	
	select tm_cz.czx_write_audit(jobId,databaseName,procedureName,'End i2b2_create_release_tablese',0,stepCt,'Done')into rtnCd;
	stepCt := stepCt + 1;
	
	    ---Cleanup OVERALL JOB if this proc is being run standalone
  IF newJobFlag = 1
  THEN
    select tm_cz.czx_end_audit (jobID, 'SUCCESS')into rtnCd;
  END IF;

  EXCEPTION
  WHEN OTHERS THEN
	v_sqlerrm := substr(SQLERRM,1,1000);
	raise notice 'error: %', v_sqlerrm;
    --Handle errors.
    select tm_cz.czx_error_handler (jobID, procedureName,v_sqlerrm)into rtnCd;
    --End Proc
    select tm_cz.czx_end_audit (jobID, 'FAIL')into rtnCd;
	
END;
$$;


ALTER FUNCTION tm_cz.i2b2_create_stage_tables(bigint) OWNER TO tm_cz;

--
-- Name: i2b2_delete_1_node(character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_delete_1_node(path character varying) RETURNS numeric
    LANGUAGE plpgsql SECURITY DEFINER
    SET search_path TO 'tm_cz', 'i2b2demodata', 'i2b2metadata', 'pg_temp'
    AS $$
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
Declare
 
	--Audit variables
	newJobFlag		integer;
	databaseName 	VARCHAR(100);
	procedureName 	VARCHAR(100);
	jobID 			numeric(18,0);
	stepCt 			numeric(18,0);
	rowCt			numeric(18,0);
	errorNumber		character varying;
	errorMessage	character varying;
	rtnCd			numeric;
	
BEGIN

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it

	IF(jobID IS NULL or jobID < 1)
	THEN
		newJobFlag := 1; -- True
		select tm_cz.cz_start_audit (procedureName, databaseName) into jobID;
	END IF;
    	
	stepCt := 0;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Start i2b2_delete_1_node: ' || path,0,stepCt,'Done') into rtnCd;
	
	if coalesce(path,'') = ''  or path = '%' 
	then 
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Path missing, no action taken',0,stepCt,'Done') into rtnCd;
		return 1;
	end if;
	
    --I2B2
	begin
    delete from i2b2demodata.observation_fact 
    where concept_cd in (select c_basecode from i2b2metadata.i2b2 where c_fullname = path);
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	get diagnostics rowCt := ROW_COUNT;	
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete data from observation_fact',rowCt,stepCt,'Done') into rtnCd;

      --CONCEPT DIMENSION
	begin
    delete from i2b2demodata.concept_dimension
    WHERE CONCEPT_PATH = path;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	get diagnostics rowCt := ROW_COUNT;	
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete data from concept_dimension',rowCt,stepCt,'Done') into rtnCd;
    
    --I2B2
	begin
    delete from i2b2metadata.i2b2
    where c_fullname = path;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	get diagnostics rowCt := ROW_COUNT;	
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete data from i2b2',rowCt,stepCt,'Done') into rtnCd;
	
	--i2b2_secure
	begin
    delete from i2b2metadata.i2b2_secure
    where c_fullname = path;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	get diagnostics rowCt := ROW_COUNT;	
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete data from i2b2_secure',rowCt,stepCt,'Done') into rtnCd;

	--concept_counts
	begin
    delete from i2b2demodata.concept_counts
    where concept_path = path;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	get diagnostics rowCt := ROW_COUNT;	
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete data from concept_counts',rowCt,stepCt,'Done') into rtnCd;
		
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'End i2b2_delete_1_node',rowCt,stepCt,'Done') into rtnCd;
	
	return 1;
  
END;

$$;


ALTER FUNCTION tm_cz.i2b2_delete_1_node(path character varying) OWNER TO tm_cz;

--
-- Name: i2b2_delete_all_nodes(character varying, numeric); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_delete_all_nodes(path character varying, currentjobid numeric DEFAULT '-1'::integer) RETURNS numeric
    LANGUAGE plpgsql SECURITY DEFINER
    SET search_path TO 'tm_cz', 'i2b2demodata', 'i2b2metadata', 'pg_temp'
    AS $$
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
Declare
 
	--Audit variables
	newJobFlag		integer;
	databaseName 	VARCHAR(100);
	procedureName 	VARCHAR(100);
	jobID 			numeric(18,0);
	stepCt 			numeric(18,0);
	rowCt			numeric(18,0);
	errorNumber		character varying;
	errorMessage	character varying;
	rtnCd			numeric;

Begin

	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;
	
	databaseName := 'TM_CZ';
	procedureName := 'I2B2_DELETE_ALL_NODES';

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it
	IF(jobID IS NULL or jobID < 1)
	THEN
		newJobFlag := 1; -- True
		select tm_cz.cz_start_audit (procedureName, databaseName) into jobID;
	END IF;
 
	if path != ''  and path != '%'
	then 
		-- observation_fact
		begin
		DELETE FROM i2b2demodata.OBSERVATION_FACT 
		WHERE 
		concept_cd IN (SELECT C_BASECODE FROM i2b2metadata.I2B2 WHERE C_FULLNAME LIKE PATH || '%' escape '`');
		get diagnostics rowCt := ROW_COUNT;
		exception
		when others then
			errorNumber := SQLSTATE;
			errorMessage := SQLERRM;
			--Handle errors.
			select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
			--End Proc
			select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
			return -16;
		end;
		stepCt := stepCt + 1;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete data for trial from I2B2DEMODATA observation_fact',rowCt,stepCt,'Done') into rtnCd;

		--CONCEPT DIMENSION
		begin
		DELETE FROM i2b2demodata.CONCEPT_DIMENSION
		WHERE CONCEPT_PATH LIKE path || '%' escape '`';
		get diagnostics rowCt := ROW_COUNT;
		exception
		when others then
			errorNumber := SQLSTATE;
			errorMessage := SQLERRM;
			--Handle errors.
			select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
			--End Proc
			select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
			return -16;
		end;
		stepCt := stepCt + 1;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete data for trial from I2B2DEMODATA concept_dimension',rowCt,stepCt,'Done') into rtnCd;
    
		--I2B2
		begin
		DELETE FROM i2b2metadata.i2b2
		WHERE C_FULLNAME LIKE PATH || '%' escape '`';
		get diagnostics rowCt := ROW_COUNT;
		exception
		when others then
			errorNumber := SQLSTATE;
			errorMessage := SQLERRM;
			--Handle errors.
			select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
			--End Proc
			select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
			return -16;
		end;
		stepCt := stepCt + 1;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete data for trial from I2B2METADATA i2b2',rowCt,stepCt,'Done') into rtnCd;
  
		--i2b2_secure
		begin
		DELETE FROM i2b2metadata.i2b2_secure
		WHERE C_FULLNAME LIKE PATH || '%' escape '`';
		get diagnostics rowCt := ROW_COUNT;
		exception
		when others then
			errorNumber := SQLSTATE;
			errorMessage := SQLERRM;
			--Handle errors.
			select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
			--End Proc
			select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
			return -16;
		end;
		stepCt := stepCt + 1;
		get diagnostics rowCt := ROW_COUNT;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete data for trial from I2B2METADATA i2b2_secure',rowCt,stepCt,'Done') into rtnCd;

		--concept_counts
		begin
		DELETE FROM i2b2demodata.concept_counts
		WHERE concept_path LIKE PATH || '%' escape '`';
		get diagnostics rowCt := ROW_COUNT;
		exception
		when others then
			errorNumber := SQLSTATE;
			errorMessage := SQLERRM;
			--Handle errors.
			select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
			--End Proc
			select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
			return -16;
		end;
		stepCt := stepCt + 1;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete data for trial from I2B2DEMODATA concept_counts',rowCt,stepCt,'Done') into rtnCd;
   
	end if;
	
	return 1;
END;

$$;


ALTER FUNCTION tm_cz.i2b2_delete_all_nodes(path character varying, currentjobid numeric) OWNER TO tm_cz;

--
-- Name: i2b2_extend_clinical_data(character varying, character varying, character varying, character varying, numeric); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_extend_clinical_data(trial_id character varying, top_node character varying, secure_study character varying DEFAULT 'N'::character varying, highlight_study character varying DEFAULT 'N'::character varying, currentjobid numeric DEFAULT '-1'::integer) RETURNS numeric
    LANGUAGE plpgsql SECURITY DEFINER
    SET search_path TO 'tm_cz', 'tm_lz', 'tm_wz', 'i2b2demodata', 'i2b2metadata', 'deapp', 'pg_temp'
    AS $$
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
-- This function is very simular to "i2b2_load_clinical_data()" except that this
-- function doesnot delete existing clinical data in the tables.
-- With this function you can "extend" the already existing clinical data for a study.
-- Note: You can not modify already existing data in de database with this function
--       It probably a good idea to combine this function with "i2b2_load_clinical_data()"
--       and do some refactoring on the way.

Declare
 
	--Audit variables
	newJobFlag		integer;
	databaseName 	VARCHAR(100);
	procedureName 	VARCHAR(100);
	jobID 			numeric(18,0);
	stepCt 			numeric(18,0);
	rowCt			numeric(18,0);
	errorNumber		character varying;
	errorMessage	character varying;
	rtnCd			integer;
	
	topNode			varchar(2000);
	topLevel		numeric(10,0);
	root_node		varchar(2000);
	root_level		integer;
	study_name		varchar(2000);
	TrialID			varchar(100);
	secureStudy		varchar(200);
	etlDate			timestamp;
	tPath			varchar(2000);
	pCount			integer;
	pExists			integer;
	rtnCode			integer;
	tText			varchar(2000);
  
	addNodes CURSOR is
	select DISTINCT leaf_node, node_name
	from  tm_wz.wt_trial_nodes a;
   
	--	cursor to define the path for delete_one_node  this will delete any nodes that are hidden after i2b2_create_concept_counts

	delNodes CURSOR is
	select distinct c_fullname 
	from  i2b2metadata.i2b2
	where c_fullname like topNode || '%' escape '`'
      and substr(c_visualattributes,2,1) = 'H';
	  
	--	cursor to determine if any leaf nodes exist in i2b2 that are not used in this reload (node changes from text to numeric or numeric to text)
	  
	delUnusedLeaf cursor is
	select l.c_fullname
	from i2b2metadata.i2b2 l
	where l.c_visualattributes like 'L%'
	  and l.c_fullname like topNode || '%' escape '`'
	  and l.c_fullname not in
		 (select t.leaf_node 
		  from tm_wz.wt_trial_nodes t
		  union
		  select m.c_fullname
		  from deapp.de_subject_sample_mapping sm
			  ,i2b2metadata.i2b2 m
		  where sm.trial_name = TrialId
		    and sm.concept_code = m.c_basecode
			and m.c_visualattributes like 'L%');
BEGIN
  
	TrialID := upper(trial_id);
	secureStudy := upper(secure_study);
	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;

	databaseName := 'TM_CZ';
	procedureName := 'I2B2_LOAD_CLINICAL_DATA';

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it

	IF(jobID IS NULL or jobID < 1)
	THEN
		newJobFlag := 1; -- True
		select tm_cz.cz_start_audit (procedureName, databaseName) into jobID;
	END IF;
    	
	stepCt := 0;
	stepCt := stepCt + 1;
	tText := 'Start i2b2_extend_clinical_data for ' || TrialId;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,tText,0,stepCt,'Done') into rtnCd;
	
	if (secureStudy not in ('Y','N') ) then
		secureStudy := 'Y';
	end if;
	
	topNode := REGEXP_REPLACE('\' || top_node || '\','(\\){2,}', '\', 'g');
	
	--	figure out how many nodes (folders) are at study name and above
	--	\Public Studies\Clinical Studies\Pancreatic_Cancer_Smith_GSE22780\: topLevel = 4, so there are 3 nodes
	--	\Public Studies\GSE12345\: topLevel = 3, so there are 2 nodes
	
	select length(topNode)-length(replace(topNode,'\','')) into topLevel;
	
	if topLevel < 3 then
		stepCt := stepCt + 1;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Path specified in top_node must contain at least 2 nodes',0,stepCt,'Done') into rtnCd;	
		select tm_cz.cz_error_handler (jobID, procedureName, '-1', 'Application raised error') into rtnCd;
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;	
	end if;	
	
	--	delete any existing data from lz_src_clinical_data and load new data
	begin
	delete from tm_lz.lz_src_clinical_data
	where study_id = TrialId;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	get diagnostics rowCt := ROW_COUNT;	
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete existing data from lz_src_clinical_data',rowCt,stepCt,'Done') into rtnCd;
	
	begin
	insert into tm_lz.lz_src_clinical_data
	(study_id
	,site_id
	,subject_id
	,visit_name
	,data_label
        ,modifier_cd
	,data_value
        ,units_cd
        ,date_timestamp
	,category_cd
	,etl_job_id
	,etl_date
	,ctrl_vocab_code)
	select study_id
		  ,site_id
		  ,subject_id
		  ,visit_name
		  ,data_label
                  ,modifier_cd
		  ,data_value
                  ,units_cd
                  ,date_timestamp
		  ,category_cd
		  ,jobId
		  ,etlDate
		  ,ctrl_vocab_code
	from tm_lz.lt_src_clinical_data;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	get diagnostics rowCt := ROW_COUNT;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Insert data into lz_src_clinical_data',rowCt,stepCt,'Done') into rtnCd;
		
	--	truncate tm_wz.wrk_clinical_data and load data from external file
	
	execute ('truncate table tm_wz.wrk_clinical_data');
	
	--	insert data from lt_src_clinical_data to tm_wz.wrk_clinical_data
	
	begin
	insert into tm_wz.wrk_clinical_data
	(study_id
	,site_id
	,subject_id
	,visit_name
	,data_label
        ,modifier_cd
	,data_value
        ,units_cd
        ,date_timestamp
	,category_cd
	,ctrl_vocab_code
	)
	select study_id
		  ,site_id
		  ,subject_id
		  ,visit_name
		  ,data_label
                  ,modifier_cd
		  ,data_value
                  ,units_cd
                  ,date_timestamp
		  ,category_cd
		  ,ctrl_vocab_code
	from tm_lz.lt_src_clinical_data;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	get diagnostics rowCt := ROW_COUNT;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Load lt_src_clinical_data to work table',rowCt,stepCt,'Done') into rtnCd;

	-- Get root_node from topNode
  
	select tm_cz.parse_nth_value(topNode, 2, '\') into root_node;
	
	select count(*) into pExists
	from i2b2metadata.table_access
	where c_name = root_node;
	
	select count(*) into pCount
	from i2b2metadata.i2b2
	where c_name = root_node;
	
	if pExists = 0 or pCount = 0 then
		select tm_cz.i2b2_add_root_node(root_node, jobId) into rtnCd;
	end if;
	
	select c_hlevel into root_level
	from i2b2metadata.table_access
	where c_name = root_node;
	
	-- Get study name from topNode
  
	select tm_cz.parse_nth_value(topNode, topLevel, '\') into study_name;
	
	--	Add any upper level nodes as needed
	
	tPath := REGEXP_REPLACE(replace(top_node,study_name,''),'(\\){2,}', '\', 'g');
	select length(tPath) - length(replace(tPath,'\','')) into pCount;

	if pCount > 2 then
		stepCt := stepCt + 1;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Adding upper-level nodes',0,stepCt,'Done') into rtnCd;
		select tm_cz.i2b2_fill_in_tree(null, tPath, jobId) into rtnCd;
	end if;

	select count(*) into pExists
	from i2b2metadata.i2b2
	where c_fullname = topNode;
	
	--	add top node for study
	
	if pExists = 0 then
		select tm_cz.i2b2_add_node(TrialId, topNode, study_name, jobId) into rtnCd;
	end if;
  
	--	Set data_type, category_path, and usubjid 
  
	update tm_wz.wrk_clinical_data
	set data_type = 'T'
	   ,category_path = replace(replace(category_cd,'_',' '),'+','\')
	   ,usubjid = REGEXP_REPLACE(TrialID || ':' || coalesce(site_id,'') || ':' || subject_id,
                   '(::){1,}', ':', 'g'); 
	 get diagnostics rowCt := ROW_COUNT;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Set columns in tm_wz.wrk_clinical_data',rowCt,stepCt,'Done') into rtnCd;

	--	Delete rows where data_value is null
  
	begin
	delete from tm_wz.wrk_clinical_data
	where data_value is null;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	get diagnostics rowCt := ROW_COUNT;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete null data_values in tm_wz.wrk_clinical_data',rowCt,stepCt,'Done') into rtnCd;
	
	--Remove Invalid pipes in the data values.
	--RULE: If Pipe is last or first, delete it
	--If it is in the middle replace with a dash

	begin
	update tm_wz.wrk_clinical_data
	set data_value = replace(trim('|' from data_value), '|', '-')
	where data_value like '%|%';
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	get diagnostics rowCt := ROW_COUNT;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Remove pipes in data_value',rowCt,stepCt,'Done') into rtnCd;
 
	--Remove invalid Parens in the data
	--They have appeared as empty pairs or only single ones.
  
	begin
	update tm_wz.wrk_clinical_data
	set data_value = replace(data_value,'(', '')
	where data_value like '%()%'
	   or data_value like '%( )%'
	   or (data_value like '%(%' and data_value NOT like '%)%');
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Remove empty parentheses 1',rowCt,stepCt,'Done') into rtnCd;
	
	begin
	update tm_wz.wrk_clinical_data
	set data_value = replace(data_value,')', '')
	where data_value like '%()%'
	   or data_value like '%( )%'
	   or (data_value like '%)%' and data_value NOT like '%(%');
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Remove empty parentheses 2',rowCt,stepCt,'Done') into rtnCd;

	--Replace the Pipes with Commas in the data_label column
	begin
	update tm_wz.wrk_clinical_data
    set data_label = replace (data_label, '|', ',')
    where data_label like '%|%';
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Replace pipes with comma in data_label',rowCt,stepCt,'Done') into rtnCd;

	--	set visit_name to null when there's only a single visit_name for the catgory
	
	begin
	update tm_wz.wrk_clinical_data tpm
	set visit_name=null
	where (tpm.category_cd) in
		  (select x.category_cd
		   from tm_wz.wrk_clinical_data x
		   group by x.category_cd
		   having count(distinct upper(x.visit_name)) = 1);
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Set single visit_name to null',rowCt,stepCt,'Done') into rtnCd;
	
	--	set data_label to null when it duplicates the last part of the category_path
	--	Remove data_label from last part of category_path when they are the same

	begin
	update tm_wz.wrk_clinical_data tpm
	--set data_label = null
	set category_path=substr(tpm.category_path,1,tm_cz.instr(tpm.category_path,'\',-2,1)-1)
	   ,category_cd=substr(tpm.category_cd,1,tm_cz.instr(tpm.category_cd,'+',-2,1)-1)
	where (tpm.category_cd, tpm.data_label) in
		  (select distinct t.category_cd
				 ,t.data_label
		   from tm_wz.wrk_clinical_data t
		   where upper(substr(t.category_path,tm_cz.instr(t.category_path,'\',-1,1)+1,length(t.category_path)-tm_cz.instr(t.category_path,'\',-1,1))) 
			     = upper(t.data_label)
		     and t.data_label is not null)
	  and tpm.data_label is not null;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Set data_label to null when found in category_path',rowCt,stepCt,'Done') into rtnCd;

	--	set visit_name to null if same as data_label
	
	begin
	update tm_wz.wrk_clinical_data t
	set visit_name=null
	where (t.category_cd, t.visit_name, t.data_label) in
	      (select distinct tpm.category_cd
				 ,tpm.visit_name
				 ,tpm.data_label
		  from tm_wz.wrk_clinical_data tpm
		  where tpm.visit_name = tpm.data_label);
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Set visit_name to null when found in data_label',rowCt,stepCt,'Done') into rtnCd;

	--	set visit_name to null if same as data_value
	
	begin
	update tm_wz.wrk_clinical_data t
	set visit_name=null
	where (t.category_cd, t.visit_name, t.data_value) in
	      (select distinct tpm.category_cd
				 ,tpm.visit_name
				 ,tpm.data_value
		  from tm_wz.wrk_clinical_data tpm
		  where tpm.visit_name = tpm.data_value);
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Set visit_name to null when found in data_value',rowCt,stepCt,'Done') into rtnCd;

	--	set visit_name to null if only DATALABEL in category_cd
	
	begin
	update tm_wz.wrk_clinical_data t
	set visit_name=null
	where t.category_cd like '%DATALABEL%'
	  and t.category_cd not like '%VISITNAME%';
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Set visit_name to null when only DATALABE in category_cd',rowCt,stepCt,'Done') into rtnCd;
	
	--	change any % to Pct and & and + to ' and ' and _ to space in data_label only
	
	begin
	update tm_wz.wrk_clinical_data
	set data_label=replace(replace(replace(replace(data_label,'%',' Pct'),'&',' and '),'+',' and '),'_',' ')
	   ,data_value=replace(replace(replace(data_value,'%',' Pct'),'&',' and '),'+',' and ')
	   ,category_cd=replace(replace(category_cd,'%',' Pct'),'&',' and ')
	   ,category_path=replace(replace(category_path,'%',' Pct'),'&',' and ');
	   exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;

  --Trim trailing and leadling spaces as well as remove any double spaces, remove space from before comma, remove trailing comma

	begin
	update tm_wz.wrk_clinical_data
	set data_label  = trim(trailing ',' from trim(replace(replace(data_label,'  ', ' '),' ,',','))),
		data_value  = trim(trailing ',' from trim(replace(replace(data_value,'  ', ' '),' ,',','))),
--		sample_type = trim(trailing ',' from trim(replace(replace(sample_type,'  ', ' '),' ,',','))),
		visit_name  = trim(trailing ',' from trim(replace(replace(visit_name,'  ', ' '),' ,',',')));
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Remove leading, trailing, double spaces',rowCt,stepCt,'Done') into rtnCd;

    --1. DETERMINE THE DATA_TYPES OF THE FIELDS
	--	replaced cursor with update, used temp table to store category_cd/data_label because correlated subquery ran too long
	
	execute ('truncate table tm_wz.wt_num_data_types');

	begin
	insert into tm_wz.wt_num_data_types
	(category_cd
	,data_label
	,visit_name
	)
    select category_cd,
           data_label,
           visit_name
    from tm_wz.wrk_clinical_data
    where data_value is not null
    group by category_cd
	        ,data_label
            ,visit_name
      having sum(tm_cz.is_numeric(data_value)) = 0;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Insert numeric data into WZ wt_num_data_types',rowCt,stepCt,'Done') into rtnCd;

	--	Check if any duplicate records of key columns (site_id, subject_id, visit_name, data_label, category_cd) for numeric data
	--	exist.  Raise error if yes
	
	execute ('truncate table tm_wz.wt_clinical_data_dups');
	
	begin
	insert into tm_wz.wt_clinical_data_dups
	(site_id
	,subject_id
	,visit_name
	,data_label
	,category_cd
        ,modifier_cd)
	select w.site_id, w.subject_id, w.visit_name, w.data_label, w.category_cd, w.modifier_cd
	from tm_wz.wrk_clinical_data w
	where exists
		 (select 1 from tm_wz.wt_num_data_types t
		 where coalesce(w.category_cd,'@') = coalesce(t.category_cd,'@')
		   and coalesce(w.data_label,'@') = coalesce(t.data_label,'@')
		   and coalesce(w.visit_name,'@') = coalesce(t.visit_name,'@')
		  )
	group by w.site_id, w.subject_id, w.visit_name, w.data_label, w.category_cd, w.modifier_cd
	having count(*) > 1;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;	
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Check for duplicate key columns',rowCt,stepCt,'Done') into rtnCd;
			  
	if rowCt > 0 then
		stepCt := stepCt + 1;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Duplicate values found in key columns',0,stepCt,'Done') into rtnCd;	
		select tm_cz.cz_error_handler (jobID, procedureName, '-1', 'Application raised error') into rtnCd;
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;	
	end if;
	
	--	check for multiple visit_names for category_cd, data_label, data_value
	
     select max(case when x.null_ct > 0 and x.non_null_ct > 0
					 then 1 else 0 end) into pCount
      from (select category_cd, data_label, data_value
				  ,sum(case when visit_name is null then 1 else 0 end) as null_ct
				  ,sum(case when visit_name is null then 0 else 1 end) as non_null_ct
			from tm_lz.lt_src_clinical_data
			where (category_cd like '%VISITNAME%' or
				   category_cd not like '%DATALABEL%')
			group by category_cd, data_label, data_value) x;
	get diagnostics rowCt := ROW_COUNT;  
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Check for multiple visit_names for category/label/value ',rowCt,stepCt,'Done') into rtnCd;
			  
	if pCount > 0 then
		stepCt := stepCt + 1;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Multiple visit names for category/label/value',0,stepCt,'Done') into rtnCd;	
		select tm_cz.cz_error_handler (jobID, procedureName, '-1', 'Application raised error') into rtnCd;
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;	
	end if;
		
	begin
	update tm_wz.wrk_clinical_data t
	set data_type='N'
	where exists
	     (select 1 from tm_wz.wt_num_data_types x
	      where coalesce(t.category_cd,'@') = coalesce(x.category_cd,'@')
			and coalesce(t.data_label,'**NULL**') = coalesce(x.data_label,'**NULL**')
			and coalesce(t.visit_name,'**NULL**') = coalesce(x.visit_name,'**NULL**')
		  );
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Updated data_type flag for numeric data_types',rowCt,stepCt,'Done') into rtnCd;

	-- Build all needed leaf nodes in one pass for both numeric and text nodes
 
	execute ('truncate table tm_wz.wt_trial_nodes');
	
	begin
	insert into tm_wz.wt_trial_nodes
	(leaf_node
	,category_cd
	,visit_name
	,data_label
	,data_value
	,data_type
	)
    select DISTINCT 
    Case 
	--	Text data_type (default node)
	When a.data_type = 'T'
	     then case when a.category_path like '%DATALABEL%' and a.category_path like '%VISITNAME%'
		      then regexp_replace(topNode || replace(replace(coalesce(a.category_path,''),'DATALABEL',coalesce(a.data_label,'')),'VISITNAME',coalesce(a.visit_name,'')) || '\' || coalesce(a.data_value,'') || '\','(\\){2,}', '\', 'g')
			  when a.category_path like '%DATALABEL%'
			  then regexp_replace(topNode || replace(coalesce(a.category_path,''),'DATALABEL',coalesce(a.data_label,'')) || '\' || coalesce(a.data_value,'') || '\','(\\){2,}', '\', 'g')
			  else REGEXP_REPLACE(topNode || coalesce(a.category_path,'') || 
                   '\'  || coalesce(a.data_label,'') || '\' || coalesce(a.data_value,'') || '\' || coalesce(a.visit_name,'') || '\',
                   '(\\){2,}', '\', 'g') 
			  end
	--	else is numeric data_type and default_node
	else case when a.category_path like '%DATALABEL%' and a.category_path like '%VISITNAME%'
		      then regexp_replace(topNode || replace(replace(coalesce(a.category_path,''),'DATALABEL',coalesce(a.data_label,'')),'VISITNAME',coalesce(a.visit_name,'')) || '\','(\\){2,}', '\', 'g')
			  when a.category_path like '%DATALABEL%'
			  then regexp_replace(topNode || replace(coalesce(a.category_path,''),'DATALABEL',coalesce(a.data_label,'')) || '\','(\\){2,}', '\', 'g')
			  else REGEXP_REPLACE(topNode || coalesce(a.category_path,'') || 
                   '\'  || coalesce(a.data_label,'') || '\' || coalesce(a.visit_name,'') || '\',
                   '(\\){2,}', '\', 'g')
			  end
	end as leaf_node,
    a.category_cd,
    a.visit_name,
	a.data_label,
	case when a.data_type = 'T' then a.data_value else null end as data_value
    ,a.data_type
	from  tm_wz.wrk_clinical_data a;
	get diagnostics rowCt := ROW_COUNT; 
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Create leaf nodes for trial',rowCt,stepCt,'Done') into rtnCd;

	--	set node_name
	
	begin
	update tm_wz.wt_trial_nodes
	set node_name=tm_cz.parse_nth_value(leaf_node,length(leaf_node)-length(replace(leaf_node,'\','')),'\');
	get diagnostics rowCt := ROW_COUNT; 
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Updated node name for leaf nodes',rowCt,stepCt,'Done') into rtnCd;

	--	insert subjects into patient_dimension if needed
	
	execute ('truncate table tm_wz.wt_subject_info');

	begin
	insert into tm_wz.wt_subject_info
	(usubjid,
     age_in_years_num,
     sex_cd,
     race_cd
    )
	select a.usubjid,
	      coalesce(max(case when upper(a.data_label) = 'AGE'
					   then case when tm_cz.is_numeric(a.data_value) = 1 then 0 else a.data_value::integer end
		               when upper(a.data_label) like '%(AGE)' 
					   then case when tm_cz.is_numeric(a.data_value) = 1 then 0 else a.data_value::integer end
					   else null end),0) as age,
		  coalesce(max(case when upper(a.data_label) = 'SEX' then a.data_value
		           when upper(a.data_label) like '%(SEX)' then a.data_value
				   when upper(a.data_label) = 'GENDER' then a.data_value
				   else null end),'Unknown') as sex,
		  max(case when upper(a.data_label) = 'RACE' then a.data_value
		           when upper(a.data_label) like '%(RACE)' then a.data_value
				   else null end) as race
	from tm_wz.wrk_clinical_data a
	group by a.usubjid;
	get diagnostics rowCt := ROW_COUNT;	
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Insert subject information into temp table',rowCt,stepCt,'Done') into rtnCd;


        --  check if there are observation in the upload which already exist in database
        --         (is there already an observation for given patient and concept)


with maps as (select td.leaf_node, usubjid from tm_wz.wrk_clinical_data cd, tm_wz.wt_trial_nodes td where cd.data_label = td.data_label)
select count(*) into pExists from i2b2demodata.observation_fact of where (of.patient_num, of.concept_cd) IN
                           (select pd.patient_num, cd.concept_cd from i2b2demodata.patient_dimension pd 
                                                                      INNER JOIN maps on pd.sourcesystem_cd = maps.usubjid
                                                                      INNER JOIN i2b2demodata.concept_dimension cd on cd.concept_path = maps.leaf_node);

        if pExists > 0 then
                  stepCt := stepCt + 1;
                  select tm_cz.cz_write_audit(jobId,databaseName, procedureName,
				'You cannot overwrite existing observations when adding.', pExists, stepCt, 'Done') into rtnCd;    
                  select tm_cz.cz_error_handler (jobID, procedureName, '-1', 'Application raised error') into rtnCd;
                  select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
                  return -16;     
        end if; 

	--  check if data to be added is of the same data-type (numeric/text) as possible already existing
        --  observations of the concept
	
	begin
		with exist_cc as (select distinct concept_cd, valtype_cd from i2b2demodata.observation_fact of
                  where of.concept_cd IN
                     (select cd.concept_cd from i2b2demodata.concept_dimension cd, tm_wz.wt_trial_nodes td
			where cd.concept_path = td.leaf_node))
                select count(*) into pCount from exist_cc, i2b2demodata.observation_fact of 
			where exist_cc.concept_cd = of.concept_cd 
				and exist_cc.valtype_cd != of.valtype_cd;
		if pCount > 0 then
                  stepCt := stepCt + 1;
                  select tm_cz.cz_write_audit(jobId,databaseName, procedureName,
				'You cannot upload different datatype for concept then already in database.', pCount, stepCt, 'Done') into rtnCd;    
                  select tm_cz.cz_error_handler (jobID, procedureName, '-1', 'Application raised error') into rtnCd;
                  select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
                  return -16;     
 		end if;
	end;


	--	insert new subjects into patient_dimension
	
	begin
	insert into i2b2demodata.patient_dimension
    (patient_num,
     sex_cd,
     age_in_years_num,
     race_cd,
     update_date,
     download_date,
     import_date,
     sourcesystem_cd
    )
    select nextval('i2b2demodata.seq_patient_num'),
		   t.sex_cd,
		   t.age_in_years_num,
		   t.race_cd,
		   current_timestamp,
		   current_timestamp,
		   current_timestamp,
		   t.usubjid
    from tm_wz.wt_subject_info t
	where t.usubjid in 
		 (select distinct cd.usubjid from tm_wz.wt_subject_info cd
		  except
		  select distinct pd.sourcesystem_cd from i2b2demodata.patient_dimension pd
		  where pd.sourcesystem_cd like TrialId || ':%');
	get diagnostics rowCt := ROW_COUNT;	
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Insert new subjects into patient_dimension',rowCt,stepCt,'Done') into rtnCd;
		
	
	--	bulk insert leaf nodes
	begin
	with ncd as (select t.leaf_node, t.node_name from tm_wz.wt_trial_nodes t)
	update i2b2demodata.concept_dimension
	set name_char=ncd.node_name
	from ncd
	where concept_path = ncd.leaf_node;
	get diagnostics rowCt := ROW_COUNT;	
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;	
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Update name_char in concept_dimension for changed names',rowCt,stepCt,'Done') into rtnCd;
	
	begin
	insert into i2b2demodata.concept_dimension
    (concept_cd
	,concept_path
	,name_char
	,update_date
	,download_date
	,import_date
	,sourcesystem_cd
	)
    select nextval('i2b2demodata.concept_id')
	     ,x.leaf_node
		 ,x.node_name
		 ,current_timestamp
		 ,current_timestamp
		 ,current_timestamp
		 ,TrialId
	from (select distinct c.leaf_node
				,c.node_name::text as node_name
		  from tm_wz.wt_trial_nodes c
		  where not exists
			(select 1 from i2b2demodata.concept_dimension x
			where c.leaf_node = x.concept_path)
		 ) x;
	get diagnostics rowCt := ROW_COUNT;	
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Inserted new leaf nodes into I2B2DEMODATA concept_dimension',rowCt,stepCt,'Done') into rtnCd;
	
			   
	begin
	insert into i2b2metadata.i2b2
    (c_hlevel
	,c_fullname
	,c_name
	,c_visualattributes
	,c_synonym_cd
	,c_facttablecolumn
	,c_tablename
	,c_columnname
	,c_dimcode
	,c_tooltip
	,update_date
	,download_date
	,import_date
	,sourcesystem_cd
	,c_basecode
	,c_operator
	,c_columndatatype
	,c_comment
	,m_applied_path
	,c_metadataxml
	)
    select (length(c.concept_path) - coalesce(length(replace(c.concept_path, '\','')),0)) / length('\') - 2 + root_level
		  ,c.concept_path
		  ,c.name_char
		  ,'LA'
		  ,'N'
		  ,'CONCEPT_CD'
		  ,'CONCEPT_DIMENSION'
		  ,'CONCEPT_PATH'
		  ,c.concept_path
		  ,c.concept_path
		  ,current_timestamp
		  ,current_timestamp
		  ,current_timestamp
		  ,c.sourcesystem_cd
		  ,c.concept_cd
		  ,'LIKE'
		  ,'T'
		  ,'trial:' || TrialID 
		  ,'@'
		  ,case when t.data_type = 'T' then null
		   else '<?xml version="1.0"?><ValueMetadata><Version>3.02</Version><CreationDateTime>08/14/2008 01:22:59</CreationDateTime><TestID></TestID><TestName></TestName><DataType>PosFloat</DataType><CodeType></CodeType><Loinc></Loinc><Flagstouse></Flagstouse><Oktousevalues>Y</Oktousevalues><MaxStringLength></MaxStringLength><LowofLowValue>0</LowofLowValue><HighofLowValue>0</HighofLowValue><LowofHighValue>100</LowofHighValue>100<HighofHighValue>100</HighofHighValue><LowofToxicValue></LowofToxicValue><HighofToxicValue></HighofToxicValue><EnumValues></EnumValues><CommentsDeterminingExclusion><Com></Com></CommentsDeterminingExclusion><UnitValues><NormalUnits>ratio</NormalUnits><EqualUnits></EqualUnits><ExcludingUnits></ExcludingUnits><ConvertingUnits><Units></Units><MultiplyingFactor></MultiplyingFactor></ConvertingUnits></UnitValues><Analysis><Enums /><Counts /><New /></Analysis></ValueMetadata>'
		   end
    from i2b2demodata.concept_dimension c
		,tm_wz.wt_trial_nodes t
    where c.concept_path = t.leaf_node
	  and not exists
		 (select 1 from i2b2metadata.i2b2 x
		  where c.concept_path = x.c_fullname);
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Inserted leaf nodes into I2B2METADATA i2b2',rowCt,stepCt,'Done') into rtnCd;

	
    --Insert into observation_fact
	
	begin
	insert into i2b2demodata.observation_fact
	(encounter_num,
     patient_num,
     concept_cd,
     start_date,
     modifier_cd,
     valtype_cd,
     tval_char,
     nval_num,
     units_cd,
     sourcesystem_cd,
     import_date,
     valueflag_cd,
     provider_id,
     location_cd,
     instance_num
	)
	select distinct c.patient_num,
		   c.patient_num,
		   i.c_basecode,
		   coalesce(a.date_timestamp, 'infinity'),
		   coalesce(a.modifier_cd, '@'),
		   a.data_type,
		   case when a.data_type = 'T' then a.data_value
				else 'E'  --Stands for Equals for numeric types
				end,
		   case when a.data_type = 'N' then a.data_value::numeric
				else null --Null for text types
				end,
                   a.units_cd,
		   a.study_id, 
		   current_timestamp, 
		   '@',
		   '@',
		   '@',
                   1
	from tm_wz.wrk_clinical_data a
		,i2b2demodata.patient_dimension c
		,tm_wz.wt_trial_nodes t
		,i2b2metadata.i2b2 i
	where a.usubjid = c.sourcesystem_cd
	  and coalesce(a.category_cd,'@') = coalesce(t.category_cd,'@')
	  and coalesce(a.data_label,'**NULL**') = coalesce(t.data_label,'**NULL**')
	  and coalesce(a.visit_name,'**NULL**') = coalesce(t.visit_name,'**NULL**')
	  and case when a.data_type = 'T' then a.data_value else '**NULL**' end = coalesce(t.data_value,'**NULL**')
	  and t.leaf_node = i.c_fullname
	  and not exists		-- don't insert if lower level node exists
		 (select 1 from tm_wz.wt_trial_nodes x
		  where x.leaf_node like t.leaf_node || '%_' escape '`')
	  and a.data_value is not null;
	get diagnostics rowCt := ROW_COUNT; 
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Insert trial into I2B2DEMODATA observation_fact',rowCt,stepCt,'Done') into rtnCd;


	-- final procs
  
	select tm_cz.i2b2_fill_in_tree(TrialId, topNode, jobID) into rtnCd;
	
	--	set sourcesystem_cd, c_comment to null if any added upper-level nodes
		

	select tm_cz.i2b2_create_concept_counts(topNode, jobID) into rtnCd;
	
	--	delete each node that is hidden after create concept counts
	
	 FOR r_delNodes in delNodes Loop

    --	deletes hidden nodes for a trial one at a time

		select tm_cz.i2b2_delete_1_node(r_delNodes.c_fullname) into rtnCd;
		stepCt := stepCt + 1;
		tText := 'Deleted node: ' || r_delNodes.c_fullname;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,tText,rowCt,stepCt,'Done') into rtnCd;

	END LOOP;  	

	select tm_cz.i2b2_create_security_for_trial(TrialId, secureStudy, jobID) into rtnCd;
	select tm_cz.i2b2_load_security_data(jobID) into rtnCd;
	
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'End i2b2_extend_clinical_data',0,stepCt,'Done') into rtnCd;
	
	---Cleanup OVERALL JOB if this proc is being run standalone
	IF newJobFlag = 1
	THEN
		select tm_cz.cz_end_audit (jobID, 'SUCCESS') into rtnCd;
	END IF;

	return 1;
/*	
	EXCEPTION
	WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
*/
END;

$$;


ALTER FUNCTION tm_cz.i2b2_extend_clinical_data(trial_id character varying, top_node character varying, secure_study character varying, highlight_study character varying, currentjobid numeric) OWNER TO tm_cz;

--
-- Name: i2b2_fill_in_tree(character varying, character varying, numeric); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_fill_in_tree(trial_id character varying, path character varying, currentjobid numeric DEFAULT '-1'::integer) RETURNS numeric
    LANGUAGE plpgsql SECURITY DEFINER
    SET search_path TO 'tm_cz', 'i2b2demodata', 'i2b2metadata', 'pg_temp'
    AS $$
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
Declare
 
	--Audit variables
	newJobFlag		integer;
	databaseName 	VARCHAR(100);
	procedureName 	VARCHAR(100);
	jobID 			numeric(18,0);
	stepCt 			numeric(18,0);
	rowCt			numeric(18,0);
	errorNumber		character varying;
	errorMessage	character varying;
	rtnCd			numeric;
	
    TrialID varchar(100);
	auditText varchar(4000);
	root_node varchar(1000);
	node_name varchar(1000);
	v_count numeric;
  
  --Get the nodes
  cNodes CURSOR for
    --Trimming off the last node as it would never need to be added.
    select distinct substr(c_fullname, 1,instr(c_fullname,'\',-2,1)) as c_fullname
    from i2b2metadata.i2b2 
    where c_fullname like path || '%' escape '`';

  
BEGIN
  TrialID := upper(trial_id);
  
    stepCt := 0;
  --Set Audit Parameters
  newJobFlag := 0; -- False (Default)
  jobID := currentJobID;

  databaseName := 'TM_CZ';
  procedureName := 'I2B2_FILL_IN_TREE';

  --Audit JOB Initialization
  --If Job ID does not exist, then this is a single procedure run and we need to create it
  IF(jobID IS NULL or jobID < 1)
  THEN
    newJobFlag := 1; -- True
    select tm_cz.cz_start_audit (procedureName, databaseName) into jobID;
  END IF;
  
  --start node with the first slash

  --Iterate through each node
  FOR r_cNodes in cNodes Loop
    root_node := '\';
    --Determine how many nodes there are
    --Iterate through, Start with 2 as one will be null from the parser
    
    for loop_counter in 2 .. (length(r_cNodes.c_fullname) - coalesce(length(replace(r_cNodes.c_fullname, '\','')),0)) / length('\')
    LOOP
		--Determine Node:
		node_name := tm_cz.parse_nth_value(r_cNodes.c_fullname, loop_counter, '\');
		root_node :=  root_node || node_name || '\';
    
        --Check if node exists. If it does not, add it.
        select count(*) into v_count 
        from i2b2metadata.i2b2
        where c_fullname = root_node;

        --If it doesn't exist, add it
        if v_count = 0 then
			auditText := 'Inserting ' || root_node;
			stepCt := stepCt + 1;
			select tm_cz.cz_write_audit(jobId,databaseName,procedureName,auditText,0,stepCt,'Done') into rtnCD;
            select tm_cz.i2b2_add_node(trial_id, root_node, node_name, jobId) into rtnCd;
        end if;
      
    END LOOP;

    --RESET VARIABLES
    root_node := '';
    node_name := '';
  END LOOP;
  
      ---Cleanup OVERALL JOB if this proc is being run standalone
  IF newJobFlag = 1
  THEN
    select tm_cz.cz_end_audit (jobID, 'SUCCESS') into rtnCD;
  END IF;
  
  return 1;

	EXCEPTION
	WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;

  
END;

$$;


ALTER FUNCTION tm_cz.i2b2_fill_in_tree(trial_id character varying, path character varying, currentjobid numeric) OWNER TO tm_cz;

--
-- Name: i2b2_hide_node(character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_hide_node(path character varying) RETURNS numeric
    LANGUAGE plpgsql SECURITY DEFINER
    SET search_path TO 'tm_cz', 'i2b2metadata', 'i2b2demodata', 'pg_temp'
    AS $$
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
declare

	--Audit variables
	newJobFlag		integer;
	databaseName 	VARCHAR(100);
	procedureName 	VARCHAR(100);
	jobID 			numeric(18,0);
	stepCt 			numeric(18,0);
	rowCt			numeric(18,0);
	errorNumber		character varying;
	errorMessage	character varying;
	rtnCd			numeric;
	
begin

    stepCt := 0;
	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;

	databaseName := 'tm_cz';
	procedureName := 'i2b2_hide_node';

  --Audit JOB Initialization
  --If Job ID does not exist, then this is a single procedure run and we need to create it
	IF(jobID IS NULL or jobID < 1)
	THEN
		newJobFlag := 1; -- True
		select tm_cz.cz_start_audit (procedureName, databaseName) into jobID;
	END IF;
	
	if path = ''  or path = '%' or path_name = ''
	then 
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Path or Path name missing, no action taken',0,stepCt,'Done') into rtnCd;
		return 1;
	end if;

	begin
	update i2b2metadata.i2b2 b
	set c_visualattributes=substr(b.c_visualattributes,1,1) || 'H' || substr(b.c_visualattributes,3,1)
	where c_fullname like path || '%' escape '`';
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	get diagnostics rowCt := ROW_COUNT;	
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Nodes hidden in i2b2metadata.i2b2',rowCt,stepCt,'Done') into rtnCd;

	
	begin
	delete from i2b2demodata.concept_counts
	where concept_path like path || '%' escape '`';
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	get diagnostics rowCt := ROW_COUNT;	
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Deleted hidden nodes from i2b2demodata.concept_counts',rowCt,stepCt,'Done') into rtnCd;

	--	reload i2b2_secure for hidden nodes
	
	select tm_cz.load_security_data(jobId) into rtnCd;
	
	return 1;
  
END;

$$;


ALTER FUNCTION tm_cz.i2b2_hide_node(path character varying) OWNER TO tm_cz;

--
-- Name: i2b2_link_additional_data(bigint); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_link_additional_data(bigint) RETURNS integer
    LANGUAGE plpgsql IMMUTABLE SECURITY DEFINER
    AS $_$
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
Declare
	--	alias for parameters
	currentJobID alias for $1;
	
	pexists			INTEGER;
	subjCt			INTEGER;
	bslash			char(1);
	rtnCd	INTEGER;
	--Audit variables
	newJobFlag INTEGER;
	databaseName VARCHAR(100);
	procedureName VARCHAR(100);
	jobID numeric(18,0);
	stepCt numeric(18,0);
	rowCount	numeric(18,0);
	v_sqlerrm	varchar(1000);

BEGIN

	stepCt := 0;
	rowCount := 0;
	bslash := '\\';

	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;

	databaseName := 'TM_CZ';
	procedureName := 'I2B2_LINK_ADDITIONAL_DATA';

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it
	IF(jobID IS NULL or jobID < 1)
	THEN
		newJobFlag := 1; -- True
		jobId := tm_cz.czx_start_audit (procedureName, databaseName);
	END IF;

	stepCt := stepCt + 1;
	SELECT tm_cz.czx_write_audit(jobId,databaseName,procedureName,'Starting ' || procedureName,0,stepCt,'Done')into rtnCd;
	
	select count(*) into pExists
	from (select t.platform, bcr.location
		  from tm_lz.lt_src_mrna_subj_samp_map t
		  left outer join biomart.bio_content_repository bcr
			   on  upper(t.platform) = upper(bcr.repository_type)
		  where bcr.location is null
		    and not exists
				(select 1 from deapp.de_gpl_info g
				 where t.platform = g.platform)) as samp;
		  
	if pExists > 0 then
		stepCt := stepCt + 1;
		SELECT tm_cz.czx_write_audit(jobId,databaseName,procedureName,'At least one platform in lt_src_mrna_subj_samp_map does not exist in bio_content_repository',0,stepCt,'Done')into rtnCd;
		return 16;
	end if;
	
	--	check to make sure subject map to patient_dimension
	
	select count(*) into subjCt
	from tm_lz.lt_src_mrna_subj_samp_map t
	where not exists
		 (select 1 from deapp.de_gpl_info g
		  where t.platform = g.platform);
	
	select count(*) into pExists
	from tm_lz.lt_src_mrna_subj_samp_map t
		,i2b2demodata.patient_dimension pd
	where REGEXP_REPLACE(t.trial_name || ':' || coalesce(t.site_id,'') || ':' || t.subject_id,
                   '(::){1,}', ':') = pd.sourcesystem_cd
	  and not exists
		 (select 1 from deapp.de_gpl_info g
		  where t.platform = g.platform);
	stepCt := stepCt + 1;
	SELECT tm_cz.czx_write_audit(jobId,databaseName,procedureName,'Check patient mapping',SQL%ROWCOUNT,stepCt,'Done')into rtnCd;
	
	if pExists < subjCt then
		stepCt := stepCt + 1;
		SELECT tm_cz.czx_write_audit(jobId,databaseName,procedureName,'One or more subject in subject_sample map file are not mapped to patients',0,stepCt,'Done')into rtnCd;
		return 16;
	end if;

	--	insert additional data into bio_content
	
	insert into biomart.bio_content
	(file_name			-- filename value in sample_cd without extension
	,repository_id		-- bio_content_repo_id
	,location			-- study_id
	,title				-- null
	,abstract			-- null
	,file_type			-- 'Data'
	,etl_id				-- null
	,etl_id_c			-- study_id
	,study_name			-- study_id
	,cel_location		-- url for site, example http://157.206.120.144:7070/CEL/BSI201_20070102/
	,cel_file_suffix	-- filename extension (everything to right of first .)
	)
	select distinct substr(t.sample_cd,1,instr(t.sample_cd,'.')-1) as file_name
	,bc.bio_content_repo_id
	,t.trial_name as location
	,null as title
	,null as abstract
	,'Data' as file_type
	,null as etl_id
	,t.trial_name as etl_id_c
	,t.trial_name as study_name
	,bc.location || '/' || t.trial_name || '/' as cel_location
	,substr(t.sample_cd,instr(t.sample_cd,'.')) as cel_file_suffix
	from tm_lz.lt_src_mrna_subj_samp_map t
		,biomart.bio_content_repository bc
	where upper(bc.repository_type) = upper(t.platform)
	  --and t.platform not in (select distinct x.platform from de_gpl_info x)
	  and not exists
		 (select 1 from biomart.bio_content x
		  where substr(t.sample_cd,1,instr(t.sample_cd,'.')-1) = x.file_name
		    and bc.bio_content_repo_id = x.repository_id
			and t.trial_name = x.location
			and bc.location || '/' || t.trial_name = x.cel_location
			and substr(t.sample_cd,instr(t.sample_cd,'.')) = x.cel_file_suffix);
	stepCt := stepCt + 1;
	SELECT tm_cz.czx_write_audit(jobId,databaseName,procedureName,'Add additional data links to bio_content',SQL%ROWCOUNT,stepCt,'Done')into rtnCd;
			
	--	insert into bio_experiment if needed

	insert into biomart.bio_experiment
	(bio_experiment_type
	,title
	,etl_id
	,accession)
	select distinct 'Experiment'
	,'Metadata not available'
	,'METADATA:' || t.trial_name
	,t.trial_name
	from tm_lz.lt_src_mrna_subj_samp_map t
	where not exists 
		(select 1 from biomart.bio_experiment x
		 where t.trial_name = x.accession);
	stepCt := stepCt + 1;
	SELECT tm_cz.czx_write_audit(jobId,databaseName,procedureName,'Added trial to bio_experiment if needed',SQL%ROWCOUNT,stepCt,'Done')into rtnCd;

	--	insert into bio_content_reference
	
	insert into biomart.bio_content_reference
	(bio_content_id
	,bio_data_id	
	,content_reference_type
	,etl_id
	,etl_id_c
	)
	select distinct bc.bio_file_content_id
	,be.bio_experiment_id
	,'Data'
	,null
	,'METADATA:' || sm.trial_name
	from tm_lz.lt_src_mrna_subj_samp_map sm
		,biomart.bio_content bc
		,biomart.bio_experiment be
	where sm.trial_name = bc.etl_id_c
	and sm.trial_name = be.accession
	and bc.file_type = 'Data'
	and not exists
	  (select 1 from biomart.bio_content_reference x
	   where x.bio_content_id = bc.bio_file_content_id
		 and x.bio_data_id = be.bio_experiment_id
		 and x.content_reference_type = 'Data');
	stepCt := stepCt + 1;
	SELECT tm_cz.czx_write_audit(jobId,databaseName,procedureName,'Added data to bio_content_reference',SQL%ROWCOUNT,stepCt,'Done')into rtnCd;

	--	insert records into de_subject_sample_mapping with dummy concept codes (-1)

	insert into deapp.de_subject_sample_mapping
	select distinct pd.patient_num as patient_id
		  ,null as site_id
		  ,t.subject_id
		  ,null as subject_type
		  ,-1 as concept_cd
		  ,-1 as assay_id
		  ,null as patient_uid
		  ,null as sample_type
		  ,null as assay_uid
		  ,t.trial_name as trial_name
		  ,null as timepoint
		  ,-1 as timepoint_cd
		  ,-1 as sample_type_cd
		  ,-1 as tissue_type_cd
		  ,t.platform as platform
		  ,-1 as platform_cd
		  ,null as tissue_type
		  ,null as data_uid
		  ,null as gpl_id
		  ,null as rbm_panel
		  ,null as sample_id
		  ,substr(t.sample_cd,1,instr(t.sample_cd,'.')-1) as sample_cd
		  ,null as category_cd
		  ,'ADDL' as source_cd
		  ,t.trial_name as omic_source_study
		  ,pd.patient_num as omic_patient_num
	from tm_lz.lt_src_mrna_subj_samp_map t
		,i2b2demodata.patient_dimension pd
	where REGEXP_REPLACE(t.trial_name || ':' || coalesce(t.site_id,'') || ':' || t.subject_id,
                   '(::){1,}', ':') = pd.sourcesystem_cd
	  and not exists
		  (select 1 from deapp.de_gpl_info g
		   where t.platform = g.platform);
	pExists := SQL%ROWCOUNT;
	stepCt := stepCt + 1;
	SELECT tm_cz.czx_write_audit(jobId,databaseName,procedureName,'Insert additional data records into de_subject_sample_mapping',SQL%ROWCOUNT,stepCt,'Done')into rtnCd;

	stepCt := stepCt + 1;
	SELECT tm_cz.czx_write_audit(jobId,databaseName,procedureName,'End ' || procedureName,0,stepCt,'Done')into rtnCd;
	
       ---Cleanup OVERALL JOB if this proc is being run standalone
	IF newJobFlag = 1
	THEN
		SELECT tm_cz.czx_end_audit (jobID, 'SUCCESS')into rtnCd;
	END IF;

	return 0;
	
	EXCEPTION
	WHEN OTHERS THEN
		v_sqlerrm := substr(SQLERRM,1,1000);
		raise notice 'error: %', v_sqlerrm;
		--Handle errors.
		SELECT tm_cz.czx_error_handler (jobID, procedureName,v_sqlerrm)into rtnCd;
		-- End Proc
		SELECT tm_cz.czx_end_audit (jobID, 'FAIL')into rtnCd;
		return 16;

END;
$_$;


ALTER FUNCTION tm_cz.i2b2_link_additional_data(bigint) OWNER TO tm_cz;

--
-- Name: i2b2_load_annotation_deapp(numeric); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_load_annotation_deapp(currentjobid numeric DEFAULT '-1'::integer) RETURNS numeric
    LANGUAGE plpgsql SECURITY DEFINER
    SET search_path TO 'tm_cz', 'deapp', 'biomart', 'pg_temp'
    AS $$
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
Declare
 
	--Audit variables
	newJobFlag		integer;
	databaseName 	VARCHAR(100);
	procedureName 	VARCHAR(100);
	jobID 			numeric(18,0);
	stepCt 			numeric(18,0);
	rowCt			numeric(18,0);
	errorNumber		character varying;
	errorMessage	character varying;
	rtnCd			integer;

	gplId			character varying;

BEGIN

	stepCt := 0;

	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;

	databaseName := 'TM_CZ';
	procedureName := 'I2B2_LOAD_ANNOTATION_DEAPP';

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it

	IF(jobID IS NULL or jobID < 1)
	THEN
		newJobFlag := 1; -- True
		select tm_cz.cz_start_audit (procedureName, databaseName) into jobID;
	END IF;

	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Starting i2b2_load_annotation_deapp',0,stepCt,'Done') into rtnCd;

	--	get GPL id from external table

	select distinct gpl_id into gplId from tm_lz.lt_src_deapp_annot;

	--	delete any existing data from annotation_deapp

	begin
	delete from tm_cz.annotation_deapp
	where gpl_id = gplId;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;

	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete existing data from annotation_deapp',rowCt,stepCt,'Done') into rtnCd;

	--	delete any existing data from deapp.de_mrna_annotation

	begin
	delete from deapp.de_mrna_annotation
	where gpl_id = gplId;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;

	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete existing data from de_mrna_annotation',rowCt,stepCt,'Done') into rtnCd;

	--	update organism for existing probesets in probeset_deapp

	begin
	with upd as (select distinct t.gpl_id, t.probe_id, t.organism from tm_lz.lt_src_deapp_annot t)
	update tm_cz.probeset_deapp
	set organism=upd.organism
	from upd
	where platform = upd.gpl_id
	  and probeset = upd.probe_id
	and exists
		 (select 1 from tm_lz.lt_src_deapp_annot x
		  where platform = x.gpl_id
		    and probeset = x.probe_id);
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;

	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Update organism in probeset_deapp',rowCt,stepCt,'Done') into rtnCd;

	--	insert any new probesets into probeset_deapp

	begin
	insert into tm_cz.probeset_deapp
	(probeset
	,organism
	,platform)
	select distinct probe_id
		  ,coalesce(organism,'Homo sapiens')
	      ,gpl_id
	from tm_lz.lt_src_deapp_annot t
	where not exists
		 (select 1 from tm_cz.probeset_deapp x
		  where t.gpl_id = x.platform
		    and t.probe_id = x.probeset
			and coalesce(t.organism,'Homo sapiens') = coalesce(x.organism,'Homo sapiens'));
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;

	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Insert new probesets into probeset_deapp',rowCt,stepCt,'Done') into rtnCd;

	--	insert data into annotation_deapp

	begin
	insert into tm_cz.annotation_deapp
	(gpl_id
	,probe_id
	,gene_symbol
	,gene_id
	,probeset_id
	,organism)
	select distinct d.gpl_id
	,d.probe_id
	,d.gene_symbol
	,d.gene_id
	,p.probeset_id
	,coalesce(d.organism,'Homo sapiens')
	from tm_lz.lt_src_deapp_annot d
	,tm_cz.probeset_deapp p
	where d.probe_id = p.probeset
	  and d.gpl_id = p.platform
	  and coalesce(d.organism,'Homo sapiens') = coalesce(p.organism,'Homo sapiens');
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;

	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Load annotation data into REFERENCE annotation_deapp',rowCt,stepCt,'Done') into rtnCd;

	--	insert data into deapp.de_mrna_annotation

	begin
	insert into deapp.de_mrna_annotation
	(gpl_id
	,probe_id
	,gene_symbol
	,gene_id
	,probeset_id
	,organism)
	select distinct d.gpl_id
	,d.probe_id
	,d.gene_symbol
	,case when d.gene_id is null then null when d.gene_id='' then null else d.gene_id::numeric end as gene_id
	,p.probeset_id
	,coalesce(d.organism,'Homo sapiens')
	from tm_lz.lt_src_deapp_annot d
	,tm_cz.probeset_deapp p
	where d.probe_id = p.probeset
	  and d.gpl_id = p.platform
	  and coalesce(d.organism,'Homo sapiens') = coalesce(p.organism,'Homo sapiens');
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;

	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Load annotation data into DEAPP de_mrna_annotation',rowCt,stepCt,'Done') into rtnCd;

	--	update gene_id if null

	begin
	with upd as (select b.bio_marker_name as gene_symbol, b.organism, min(b.primary_external_id::numeric) as gene_id 
				 from biomart.bio_marker b
				 where upper(b.bio_marker_type) = 'GENE'
				 group by b.bio_marker_name, b.organism)
	update deapp.de_mrna_annotation a
	set gene_id=upd.gene_id
	from upd
	where a.gpl_id = gplId
	  and a.gene_id is null
	  and a.gene_symbol is not null
	  and a.gene_symbol = upd.gene_symbol
	  and upper(a.organism) = upper(upd.organism)
	  and exists
	      (select 1 from biomart.bio_marker x
              where a.gene_symbol = x.bio_marker_name
                and upper(x.organism) = upper(a.organism)
             	and upper(x.bio_marker_type) = 'GENE');
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Updated missing gene_id in de_mrna_annotation',rowCt,stepCt,'Done') into rtnCd;

	--	update gene_symbol if null

	begin
	with upd as (select b.primary_external_id::numeric as gene_id, b.organism, min(b.bio_marker_name) as gene_symbol
				 from biomart.bio_marker b
				 where upper(b.bio_marker_type) = 'GENE'
				 group by b.primary_external_id, b.organism)
	update deapp.de_mrna_annotation a
	set gene_symbol=upd.gene_symbol
	from upd
	where a.gpl_id = gplId
	  and a.gene_symbol is null
	  and a.gene_id is not null
	  and a.gene_id = upd.gene_id
	  and upper(a.organism) = upper(upd.organism)
	  and exists
	      (select 1 from biomart.bio_marker x
              where a.gene_symbol = x.bio_marker_name
                and upper(x.organism) = upper(a.organism)
             	and upper(x.bio_marker_type) = 'GENE');
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Updated missing gene_symbol in de_mrna_annotation',rowCt,stepCt,'Done') into rtnCd;

	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'End i2b2_load_annotation_deapp',0,stepCt,'Done') into rtnCd;

       --Cleanup OVERALL JOB if this proc is being run standalone
	IF newJobFlag = 1
	THEN
		select tm_cz.cz_end_audit (jobID, 'SUCCESS') into rtnCd;
	END IF;

	return 1;

END;

$$;


ALTER FUNCTION tm_cz.i2b2_load_annotation_deapp(currentjobid numeric) OWNER TO tm_cz;

--
-- Name: i2b2_load_chrom_region(character varying, character varying, character varying, character varying, numeric); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_load_chrom_region(platform_title character varying DEFAULT ''::character varying, data_type character varying DEFAULT 'ACGH'::character varying, genome_release character varying DEFAULT ''::character varying, gene_annotation_id character varying DEFAULT ''::character varying, currentjobid numeric DEFAULT '-1'::integer) RETURNS numeric
    LANGUAGE plpgsql
    SET search_path TO 'tm_cz', 'deapp'
    AS $$

Declare
	--Audit variables
	newJobFlag		integer;
	databaseName 		VARCHAR(100);
	procedureName 		VARCHAR(100);
	jobID 			numeric(18,0);
	stepCt 			numeric(18,0);
	rowCt			numeric(18,0);
	errorNumber		character varying;
	errorMessage	character varying;
	rtnCd			integer;

	gplId			character varying;
    marker_type		character varying;
	organismId		character varying;
	sqlText			varchar(1000);

BEGIN
	stepCt := 0;

	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;

	databaseName := 'TM_CZ';
	procedureName := 'I2B2_LOAD_CHROM_REGION';

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it

	IF(jobID IS NULL or jobID < 1)
	THEN
		newJobFlag := 1; -- True
		select tm_cz.cz_start_audit (procedureName, databaseName) into jobID;
	END IF;

	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Starting i2b2_load_chrom_region',0,stepCt,'Done') into rtnCd;

-- The data should already be in the landing zone (tm_lz.lt_chromosomal_region)


-- We now do some basic check's:
	-- + is chromosomal_region already in deapp.de_chromosomal_region (gpl_id/region_name)
	--   if true then remove these lines?
	-- + ...


-- insert region definitions into deapp-schema

	-- First remove previous definitions for gpl_id
	select distinct gpl_id INTO gplId FROM tm_lz.lt_chromosomal_region;

	begin
	delete from deapp.de_chromosomal_region
	where gpl_id = gplId;
	get diagnostics rowCt := ROW_COUNT;	
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;

	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete existing data from deapp.de_chromosomal_region for plaform: ' || gplId,rowCt,stepCt,'Done') into rtnCd;

	begin
	delete from deapp.de_gpl_info
	where platform = gplId;
	get diagnostics rowCt := ROW_COUNT;	
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;

	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete existing data from deapp.de_gpl_info for platform: ' || gplID,rowCt,stepCt,'Done') into rtnCd;


    -- Insert platform into deapp.de_gpl_info

    -- Derive marker_type from data_type argument (defaults to CHROMOSOME_REGION_ACGH)
	if (upper(data_type) = 'RNASEQ')
	then
		marker_type := 'RNASEQ_RCNT';
	else
		marker_type := 'Chromosomal';
	end if;

	if (length(platform_title) = 0)
	then
		platform_title = gplId;
	end if;

	select distinct organism INTO organismId FROM tm_lz.lt_chromosomal_region;
	begin
	insert into deapp.de_gpl_info 
	(
		platform
		, title
		, organism
		, annotation_date
		, marker_type
		, genome_build
		, gene_annotation_id
	)
	values (gplId, platform_title, organismId, current_timestamp, marker_type, genome_release, gene_annotation_id);
	get diagnostics rowCt := ROW_COUNT;	
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;

	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Load platform info into deapp.de_gpl_info for platform: ' || gplID,rowCt,stepCt,'Done') into rtnCd;


	-- Next insert the new definitions
	BEGIN	
	insert into deapp.de_chromosomal_region 
  	(     GPL_ID
	    , REGION_NAME
	    , CHROMOSOME
	    , START_BP
	    , END_BP
	    , NUM_PROBES
	    , CYTOBAND
	    , GENE_SYMBOL
	    , GENE_ID
	    , ORGANISM
	)
	select    lz.GPL_ID
	        , lz.REGION_NAME
	    	, lz.CHROMOSOME
	    	, lz.START_BP
	    	, lz.END_BP
	    	, lz.NUM_PROBES
	    	, lz.CYTOBAND
	    	, lz.GENE_SYMBOL
	    	, lz.GENE_ID
	    	, lz.ORGANISM
	from tm_lz.lt_chromosomal_region lz;
	get diagnostics rowCt := ROW_COUNT;	
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;

	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Load chromosomal region data into deapp.de_chromosomal_region for platform: ' || gplId,rowCt,stepCt,'Done') into rtnCd;


--	update gene_id if null
	
	begin
	with upd as (select b.bio_marker_name as gene_symbol, b.organism, min(b.primary_external_id::numeric) as gene_id 
			 from biomart.bio_marker b
			 where upper(b.bio_marker_type) = 'GENE'
			 group by b.bio_marker_name, b.organism)
	update deapp.de_chromosomal_region a
	set gene_id=upd.gene_id
	from upd
	where a.gpl_id = gplId
	      and a.gene_id is null
	      and a.gene_symbol is not null
	      and a.gene_symbol = upd.gene_symbol
	      and upper(a.organism) = upper(upd.organism)
	      and exists
		 (select 1 from biomart.bio_marker x
		  where a.gene_symbol = x.bio_marker_name
			and upper(x.organism) = upper(a.organism)
			and upper(x.bio_marker_type) = 'GENE');
	get diagnostics rowCt := ROW_COUNT;	
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;		
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Updated missing gene_id in de_chromosomal_region',rowCt,stepCt,'Done') into rtnCd;


	--	update gene_symbol if null
	
	begin
	with upd as (select b.primary_external_id::numeric as gene_id, b.organism, min(b.bio_marker_name) as gene_symbol
			 from biomart.bio_marker b
			 where upper(b.bio_marker_type) = 'GENE'
			 group by b.primary_external_id, b.organism)
	update deapp.de_chromosomal_region a
	set gene_symbol=upd.gene_symbol
	from upd
	where a.gpl_id = gplId
	      and a.gene_symbol is null
	      and a.gene_id is not null
	      and a.gene_id = upd.gene_id
	      and a.organism = upd.organism
	      and exists
		 (select 1 from biomart.bio_marker x
		  where a.gene_id::varchar = x.primary_external_id
			and upper(x.organism) = upper(a.organism)
			and upper(x.bio_marker_type) = 'GENE');
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;	
	end;		
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Updated missing gene_symbol in de_chromosomal_region',rowCt,stepCt,'Done') into rtnCd;


-- wrapping up
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'End i2b2_load_chrom_region',0,stepCt,'Done') into rtnCd;

       ---Cleanup OVERALL JOB if this proc is being run standalone
	IF newJobFlag = 1
	THEN
		select tm_cz.cz_end_audit (jobID, 'SUCCESS') into rtnCd;
	END IF;

	return 1;

END;

$$;


ALTER FUNCTION tm_cz.i2b2_load_chrom_region(platform_title character varying, data_type character varying, genome_release character varying, gene_annotation_id character varying, currentjobid numeric) OWNER TO tm_cz;

--
-- Name: i2b2_load_clinical_data(character varying, character varying, character varying, character varying, numeric); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_load_clinical_data(trial_id character varying, top_node character varying, secure_study character varying DEFAULT 'N'::character varying, highlight_study character varying DEFAULT 'N'::character varying, currentjobid numeric DEFAULT '-1'::integer) RETURNS numeric
    LANGUAGE plpgsql SECURITY DEFINER
    SET search_path TO 'tm_cz', 'tm_lz', 'tm_wz', 'i2b2demodata', 'i2b2metadata', 'deapp', 'pg_temp'
    AS $$
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
Declare
 
	--Audit variables
	newJobFlag		integer;
	databaseName 	VARCHAR(100);
	procedureName 	VARCHAR(100);
	jobID 			numeric(18,0);
	stepCt 			numeric(18,0);
	rowCt			numeric(18,0);
	errorNumber		character varying;
	errorMessage	character varying;
	rtnCd			integer;
	
	topNode			varchar(2000);
	topLevel		numeric(10,0);
	root_node		varchar(2000);
	root_level		integer;
	study_name		varchar(2000);
	TrialID			varchar(100);
	secureStudy		varchar(200);
	etlDate			timestamp;
	tPath			varchar(2000);
	pCount			integer;
	pExists			integer;
	rtnCode			integer;
	tText			varchar(2000);
  
	addNodes CURSOR is
	select DISTINCT leaf_node, node_name
	from  tm_wz.wt_trial_nodes a;
   
	--	cursor to define the path for delete_one_node  this will delete any nodes that are hidden after i2b2_create_concept_counts

	delNodes CURSOR is
	select distinct c_fullname 
	from  i2b2metadata.i2b2
	where c_fullname like topNode || '%' escape '`'
      and substr(c_visualattributes,2,1) = 'H';
	  
	--	cursor to determine if any leaf nodes exist in i2b2 that are not used in this reload (node changes from text to numeric or numeric to text)
	  
	delUnusedLeaf cursor is
	select l.c_fullname
	from i2b2metadata.i2b2 l
	where l.c_visualattributes like 'L%'
	  and l.c_fullname like topNode || '%' escape '`'
	  and l.c_fullname not in
		 (select t.leaf_node 
		  from tm_wz.wt_trial_nodes t
		  union
		  select m.c_fullname
		  from deapp.de_subject_sample_mapping sm
			  ,i2b2metadata.i2b2 m
		  where sm.trial_name = TrialId
		    and sm.concept_code = m.c_basecode
			and m.c_visualattributes like 'L%');
			
	-- added by Cognizant for requirement 3 and 4 under #1
    uploadI2b2 cursor is
    select category_cd,display_value,display_label,display_unit from
    tm_lz.lt_src_display_mapping group by category_cd,display_value,display_label,display_unit;
BEGIN
  
	TrialID := upper(trial_id);
	secureStudy := upper(secure_study);
	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;
	
	

	databaseName := 'TM_CZ';
	procedureName := 'I2B2_LOAD_CLINICAL_DATA';
	
	
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Start i2b2_load_clinical_data ',0,stepCt,'Done') into rtnCd;

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it

	IF(jobID IS NULL or jobID < 1)
	THEN
		newJobFlag := 1; -- True
		select tm_cz.cz_start_audit (procedureName, databaseName) into jobID;
	END IF;
    	
	stepCt := 0;
	stepCt := stepCt + 1;
	tText := 'Start i2b2_load_clinical_data for ' || TrialId;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,tText,0,stepCt,'Done') into rtnCd;
	
	if (secureStudy not in ('Y','N') ) then
		secureStudy := 'Y';
	end if;
	
	topNode := REGEXP_REPLACE('\' || top_node || '\','(\\){2,}', '\', 'g');
	
	--	figure out how many nodes (folders) are at study name and above
	--	\Public Studies\Clinical Studies\Pancreatic_Cancer_Smith_GSE22780\: topLevel = 4, so there are 3 nodes
	--	\Public Studies\GSE12345\: topLevel = 3, so there are 2 nodes
	
	select length(topNode)-length(replace(topNode,'\','')) into topLevel;
	
	if topLevel < 3 then
		stepCt := stepCt + 1;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Path specified in top_node must contain at least 2 nodes',0,stepCt,'Done') into rtnCd;
		select tm_cz.cz_error_handler (jobID, procedureName, '-1', 'Application raised error') into rtnCd;
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end if;

	--	delete any existing data from lz_src_clinical_data and load new data
	begin
	delete from tm_lz.lz_src_clinical_data
	where study_id = TrialId;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	get diagnostics rowCt := ROW_COUNT;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete existing data from lz_src_clinical_data',rowCt,stepCt,'Done') into rtnCd;
	
	begin
	insert into tm_lz.lz_src_clinical_data
	(study_id
	,site_id
	,subject_id
	,visit_name
	,data_label
        ,modifier_cd
	,data_value
        ,units_cd
        ,date_timestamp
	,category_cd
	,etl_job_id
	,etl_date
	,ctrl_vocab_code)
	select study_id
		  ,site_id
		  ,subject_id
		  ,visit_name
		  ,data_label
                  ,modifier_cd
		  ,data_value
                  ,units_cd
                  ,date_timestamp
		  ,category_cd
		  ,jobId
		  ,etlDate
		  ,ctrl_vocab_code
	from tm_lz.lt_src_clinical_data;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	get diagnostics rowCt := ROW_COUNT;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Insert data into lz_src_clinical_data',rowCt,stepCt,'Done') into rtnCd;
		
	--	truncate tm_wz.wrk_clinical_data and load data from external file
	
	execute ('truncate table tm_wz.wrk_clinical_data');
	
	--	insert data from lt_src_clinical_data to tm_wz.wrk_clinical_data
	
	begin
	insert into tm_wz.wrk_clinical_data
	(study_id
	,site_id
	,subject_id
	,visit_name
	,data_label
        ,modifier_cd
	,data_value
        ,units_cd
        ,date_timestamp
	,category_cd
	,ctrl_vocab_code
	)
	select study_id
		  ,site_id
		  ,subject_id
		  ,visit_name
		  ,data_label
                  ,modifier_cd
		  ,data_value
                  ,units_cd
                  ,date_timestamp
		  ,category_cd
		  ,ctrl_vocab_code
	from tm_lz.lt_src_clinical_data;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	get diagnostics rowCt := ROW_COUNT;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Load lt_src_clinical_data to work table',rowCt,stepCt,'Done') into rtnCd;

	-- Get root_node from topNode
  
	select tm_cz.parse_nth_value(topNode, 2, '\') into root_node;
	
	select count(*) into pExists
	from i2b2metadata.table_access
	where c_name = root_node;
	
	select count(*) into pCount
	from i2b2metadata.i2b2
	where c_name = root_node;
	
	if pExists = 0 or pCount = 0 then
		select tm_cz.i2b2_add_root_node(root_node, jobId) into rtnCd;
	end if;
	
	select c_hlevel into root_level
	from i2b2metadata.table_access
	where c_name = root_node;
	
	-- Get study name from topNode
  
	select tm_cz.parse_nth_value(topNode, topLevel, '\') into study_name;
	
	--	Add any upper level nodes as needed
	
	tPath := REGEXP_REPLACE(replace(top_node,study_name,''),'(\\){2,}', '\', 'g');
	select length(tPath) - length(replace(tPath,'\','')) into pCount;

	if pCount > 2 then
		stepCt := stepCt + 1;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Adding upper-level nodes',0,stepCt,'Done') into rtnCd;
		select tm_cz.i2b2_fill_in_tree(null, tPath, jobId) into rtnCd;
	end if;

	select count(*) into pExists
	from i2b2metadata.i2b2
	where c_fullname = topNode;
	
	--	add top node for study
	
	if pExists = 0 then
		select tm_cz.i2b2_add_node(TrialId, topNode, study_name, jobId) into rtnCd;
	end if;
  
	--	Set data_type, category_path, and usubjid 
  
	update tm_wz.wrk_clinical_data
	set data_type = 'T'
	   ,category_path = replace(replace(category_cd,'_',' '),'+','\')
	   ,usubjid = REGEXP_REPLACE(TrialID || ':' || coalesce(site_id,'') || ':' || subject_id,
                   '(::){1,}', ':', 'g'); 
	 get diagnostics rowCt := ROW_COUNT;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Set columns in tm_wz.wrk_clinical_data',rowCt,stepCt,'Done') into rtnCd;

	--	Delete rows where data_value is null
  
	begin
	delete from tm_wz.wrk_clinical_data
	where data_value is null;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	get diagnostics rowCt := ROW_COUNT;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete null data_values in tm_wz.wrk_clinical_data',rowCt,stepCt,'Done') into rtnCd;
	
	--Remove Invalid pipes in the data values.
	--RULE: If Pipe is last or first, delete it
	--If it is in the middle replace with a dash

	begin
	update tm_wz.wrk_clinical_data
	set data_value = replace(trim('|' from data_value), '|', '-')
	where data_value like '%|%';
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	get diagnostics rowCt := ROW_COUNT;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Remove pipes in data_value',rowCt,stepCt,'Done') into rtnCd;
 
	--Remove invalid Parens in the data
	--They have appeared as empty pairs or only single ones.
  
	begin
	update tm_wz.wrk_clinical_data
	set data_value = replace(data_value,'(', '')
	where data_value like '%()%'
	   or data_value like '%( )%'
	   or (data_value like '%(%' and data_value NOT like '%)%');
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Remove empty parentheses 1',rowCt,stepCt,'Done') into rtnCd;
	
	begin
	update tm_wz.wrk_clinical_data
	set data_value = replace(data_value,')', '')
	where data_value like '%()%'
	   or data_value like '%( )%'
	   or (data_value like '%)%' and data_value NOT like '%(%');
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Remove empty parentheses 2',rowCt,stepCt,'Done') into rtnCd;

	--Replace the Pipes with Commas in the data_label column
	begin
	update tm_wz.wrk_clinical_data
    set data_label = replace (data_label, '|', ',')
    where data_label like '%|%';
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Replace pipes with comma in data_label',rowCt,stepCt,'Done') into rtnCd;

	--	set visit_name to null when there's only a single visit_name for the catgory
	
	begin
	update tm_wz.wrk_clinical_data tpm
	set visit_name=null
	where (tpm.category_cd) in
		  (select x.category_cd
		   from tm_wz.wrk_clinical_data x
		   group by x.category_cd
		   having count(distinct upper(x.visit_name)) = 1);
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Set single visit_name to null',rowCt,stepCt,'Done') into rtnCd;
	
	--	set data_label to null when it duplicates the last part of the category_path
	--	Remove data_label from last part of category_path when they are the same

	begin
	update tm_wz.wrk_clinical_data tpm
	--set data_label = null
	set category_path=substr(tpm.category_path,1,tm_cz.instr(tpm.category_path,'\',-2,1)-1)
	   ,category_cd=substr(tpm.category_cd,1,tm_cz.instr(tpm.category_cd,'+',-2,1)-1)
	where tm_cz.instr(tpm.category_path,'\',-1,1) > 0 and (tpm.category_cd, tpm.data_label) in
		  (select distinct t.category_cd
				 ,t.data_label
		   from tm_wz.wrk_clinical_data t
		   where upper(substr(t.category_path,tm_cz.instr(t.category_path,'\',-1,1)+1,length(t.category_path)-tm_cz.instr(t.category_path,'\',-1,1))) 
			     = upper(t.data_label)
		     and t.data_label is not null)
	  and tpm.data_label is not null;
        update tm_wz.wrk_clinical_data tpm
        set data_label = null
        where tm_cz.instr(tpm.category_path,'\',-1,1) = 0 and (tpm.category_cd, tpm.data_label) in
                  (select distinct t.category_cd
                                 ,t.data_label
                   from tm_wz.wrk_clinical_data t
                   where upper(substr(t.category_path,tm_cz.instr(t.category_path,'\',-1,1)+1,length(t.category_path)-tm_cz.instr(t.category_path,'\',-1,1)))
                             = upper(t.data_label)
                     and t.data_label is not null)
          and tpm.data_label is not null;
        get diagnostics rowCt := ROW_COUNT;
        exception
        when others then
                errorNumber := SQLSTATE;
                errorMessage := SQLERRM;
                --Handle errors.
                select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
                --End Proc
                select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
                return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Set data_label to null when found in category_path',rowCt,stepCt,'Done') into rtnCd;

	--	set visit_name to null if same as data_label
	
	begin
	update tm_wz.wrk_clinical_data t
	set visit_name=null
	where (t.category_cd, t.visit_name, t.data_label) in
	      (select distinct tpm.category_cd
				 ,tpm.visit_name
				 ,tpm.data_label
		  from tm_wz.wrk_clinical_data tpm
		  where tpm.visit_name = tpm.data_label);
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Set visit_name to null when found in data_label',rowCt,stepCt,'Done') into rtnCd;

	--	set visit_name to null if same as data_value
	
	begin
	update tm_wz.wrk_clinical_data t
	set visit_name=null
	where (t.category_cd, t.visit_name, t.data_value) in
	      (select distinct tpm.category_cd
				 ,tpm.visit_name
				 ,tpm.data_value
		  from tm_wz.wrk_clinical_data tpm
		  where tpm.visit_name = tpm.data_value);
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Set visit_name to null when found in data_value',rowCt,stepCt,'Done') into rtnCd;

	--	set visit_name to null if only DATALABEL in category_cd
	
	begin
	update tm_wz.wrk_clinical_data t
	set visit_name=null
	where t.category_cd like '%DATALABEL%'
	  and t.category_cd not like '%VISITNAME%';
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Set visit_name to null when only DATALABE in category_cd',rowCt,stepCt,'Done') into rtnCd;
	
	--	change any % to Pct and & and + to ' and ' and _ to space in data_label only
	
	begin
	update tm_wz.wrk_clinical_data
	set data_label=replace(replace(replace(replace(data_label,'%',' Pct'),'&',' and '),'+',' and '),'_',' ')
	   ,data_value=replace(replace(replace(data_value,'%',' Pct'),'&',' and '),'+',' and ')
	   ,category_cd=replace(replace(category_cd,'%',' Pct'),'&',' and ')
	   ,category_path=replace(replace(category_path,'%',' Pct'),'&',' and ');
	   exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;

  --Trim trailing and leadling spaces as well as remove any double spaces, remove space from before comma, remove trailing comma

	begin
	update tm_wz.wrk_clinical_data
	set data_label  = trim(trailing ',' from trim(replace(replace(data_label,'  ', ' '),' ,',','))),
		data_value  = trim(trailing ',' from trim(replace(replace(data_value,'  ', ' '),' ,',','))),
--		sample_type = trim(trailing ',' from trim(replace(replace(sample_type,'  ', ' '),' ,',','))),
		visit_name  = trim(trailing ',' from trim(replace(replace(visit_name,'  ', ' '),' ,',',')));
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Remove leading, trailing, double spaces',rowCt,stepCt,'Done') into rtnCd;

    --1. DETERMINE THE DATA_TYPES OF THE FIELDS
	--	replaced cursor with update, used temp table to store category_cd/data_label because correlated subquery ran too long
	
	execute ('truncate table tm_wz.wt_num_data_types');

	begin
	insert into tm_wz.wt_num_data_types
	(category_cd
	,data_label
	,visit_name
	)
    select category_cd,
           data_label,
           visit_name
    from tm_wz.wrk_clinical_data
    where data_value is not null
    group by category_cd
	        ,data_label
            ,visit_name
      having sum(tm_cz.is_numeric(data_value)) = 0;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Insert numeric data into WZ wt_num_data_types',rowCt,stepCt,'Done') into rtnCd;

	--	Check if any duplicate records of key columns (site_id, subject_id, visit_name, data_label, category_cd) for numeric data
	--	exist.  Raise error if yes
	
	execute ('truncate table tm_wz.wt_clinical_data_dups');
	
	begin
	insert into tm_wz.wt_clinical_data_dups
	(site_id
	,subject_id
	,visit_name
	,data_label
	,category_cd
        ,modifier_cd)
	select w.site_id, w.subject_id, w.visit_name, w.data_label, w.category_cd, w.modifier_cd
	from tm_wz.wrk_clinical_data w
	where exists
		 (select 1 from tm_wz.wt_num_data_types t
		 where coalesce(w.category_cd,'@') = coalesce(t.category_cd,'@')
		   and coalesce(w.data_label,'@') = coalesce(t.data_label,'@')
		   and coalesce(w.visit_name,'@') = coalesce(t.visit_name,'@')
		  )
	group by w.site_id, w.subject_id, w.visit_name, w.data_label, w.category_cd, w.modifier_cd
	having count(*) > 1;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Check for duplicate key columns',rowCt,stepCt,'Done') into rtnCd;
			  
	if rowCt > 0 then
		stepCt := stepCt + 1;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Duplicate values found in key columns',0,stepCt,'Done') into rtnCd;
		select tm_cz.cz_error_handler (jobID, procedureName, '-1', 'Application raised error') into rtnCd;
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end if;
	
	--	check for multiple visit_names for category_cd, data_label, data_value
	
     select max(case when x.null_ct > 0 and x.non_null_ct > 0
					 then 1 else 0 end) into pCount
      from (select category_cd, data_label, data_value
				  ,sum(case when visit_name is null then 1 else 0 end) as null_ct
				  ,sum(case when visit_name is null then 0 else 1 end) as non_null_ct
			from tm_lz.lt_src_clinical_data
			where (category_cd like '%VISITNAME%' or
				   category_cd not like '%DATALABEL%')
			group by category_cd, data_label, data_value) x;
	get diagnostics rowCt := ROW_COUNT;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Check for multiple visit_names for category/label/value ',rowCt,stepCt,'Done') into rtnCd;
			  
	if pCount > 0 then
		stepCt := stepCt + 1;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Multiple visit names for category/label/value',0,stepCt,'Done') into rtnCd;
		select tm_cz.cz_error_handler (jobID, procedureName, '-1', 'Application raised error') into rtnCd;
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end if;
		
	begin
	update tm_wz.wrk_clinical_data t
	set data_type='N'
	where exists
	     (select 1 from tm_wz.wt_num_data_types x
	      where coalesce(t.category_cd,'@') = coalesce(x.category_cd,'@')
			and coalesce(t.data_label,'**NULL**') = coalesce(x.data_label,'**NULL**')
			and coalesce(t.visit_name,'**NULL**') = coalesce(x.visit_name,'**NULL**')
		  );
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Updated data_type flag for numeric data_types',rowCt,stepCt,'Done') into rtnCd;

	-- Build all needed leaf nodes in one pass for both numeric and text nodes
 
	execute ('truncate table tm_wz.wt_trial_nodes');
	
	begin
	insert into tm_wz.wt_trial_nodes
	(leaf_node
	,category_cd
	,visit_name
	,data_label
	,data_value
	,data_type
	)
    select DISTINCT 
    Case 
	--	Text data_type (default node)
	When a.data_type = 'T'
	     then case when a.category_path like '%DATALABEL%' and a.category_path like '%VISITNAME%'
		      then regexp_replace(topNode || replace(replace(coalesce(a.category_path,''),'DATALABEL',coalesce(a.data_label,'')),'VISITNAME',coalesce(a.visit_name,'')) || '\' || coalesce(a.data_value,'') || '\','(\\){2,}', '\', 'g')
			  when a.category_path like '%DATALABEL%'
			  then regexp_replace(topNode || replace(coalesce(a.category_path,''),'DATALABEL',coalesce(a.data_label,'')) || '\' || coalesce(a.data_value,'') || '\','(\\){2,}', '\', 'g')
			  else REGEXP_REPLACE(topNode || coalesce(a.category_path,'') || 
                   '\'  || coalesce(a.data_label,'') || '\' || coalesce(a.data_value,'') || '\' || coalesce(a.visit_name,'') || '\',
                   '(\\){2,}', '\', 'g') 
			  end
	--	else is numeric data_type and default_node
	else case when a.category_path like '%DATALABEL%' and a.category_path like '%VISITNAME%'
		      then regexp_replace(topNode || replace(replace(coalesce(a.category_path,''),'DATALABEL',coalesce(a.data_label,'')),'VISITNAME',coalesce(a.visit_name,'')) || '\','(\\){2,}', '\', 'g')
			  when a.category_path like '%DATALABEL%'
			  then regexp_replace(topNode || replace(coalesce(a.category_path,''),'DATALABEL',coalesce(a.data_label,'')) || '\','(\\){2,}', '\', 'g')
			  else REGEXP_REPLACE(topNode || coalesce(a.category_path,'') || 
                   '\'  || coalesce(a.data_label,'') || '\' || coalesce(a.visit_name,'') || '\',
                   '(\\){2,}', '\', 'g')
			  end
	end as leaf_node,
    a.category_cd,
    a.visit_name,
	a.data_label,
	case when a.data_type = 'T' then a.data_value else null end as data_value
    ,a.data_type
	from  tm_wz.wrk_clinical_data a;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Create leaf nodes for trial',rowCt,stepCt,'Done') into rtnCd;

	--	set node_name
	
	begin
	update tm_wz.wt_trial_nodes
	set node_name=tm_cz.parse_nth_value(leaf_node,length(leaf_node)-length(replace(leaf_node,'\','')),'\');
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Updated node name for leaf nodes',rowCt,stepCt,'Done') into rtnCd;

	--	insert subjects into patient_dimension if needed
	
	execute ('truncate table tm_wz.wt_subject_info');

	begin
	insert into tm_wz.wt_subject_info
	(usubjid,
     age_in_years_num,
     sex_cd,
     race_cd
    )
	select a.usubjid,
	      max(case when upper(a.data_label) = 'AGE'
				   then case when tm_cz.is_numeric(a.data_value) = 1 then null else round(a.data_value::numeric) end
		           when upper(a.data_label) like '%(AGE)' 
				   then case when tm_cz.is_numeric(a.data_value) = 1 then null else round(a.data_value::numeric) end
				   else null end) as age,
		  max(case when upper(a.data_label) = 'SEX' then a.data_value
		           when upper(a.data_label) like '%(SEX)' then a.data_value
				   when upper(a.data_label) = 'GENDER' then a.data_value
				   else null end) as sex,
		  max(case when upper(a.data_label) = 'RACE' then a.data_value
		           when upper(a.data_label) like '%(RACE)' then a.data_value
				   else null end) as race
	from tm_wz.wrk_clinical_data a
	group by a.usubjid;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Insert subject information into temp table',rowCt,stepCt,'Done') into rtnCd;

	--	Delete dropped subjects from patient_dimension if they do not exist in de_subject_sample_mapping
	
	begin
	delete from i2b2demodata.patient_dimension
	where sourcesystem_cd in
		 (select distinct pd.sourcesystem_cd from i2b2demodata.patient_dimension pd
		  where pd.sourcesystem_cd like TrialId || ':%'
		  except 
		  select distinct cd.usubjid from tm_wz.wrk_clinical_data cd)
	  and patient_num not in
		  (select distinct sm.patient_id from deapp.de_subject_sample_mapping sm);
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete dropped subjects from patient_dimension',rowCt,stepCt,'Done') into rtnCd;

	--	update patients with changed information
	begin
	with nsi as (select t.usubjid, t.sex_cd, t.age_in_years_num, t.race_cd from tm_wz.wt_subject_info t) 
	update i2b2demodata.patient_dimension
	set sex_cd=nsi.sex_cd
	   ,age_in_years_num=nsi.age_in_years_num
	   ,race_cd=nsi.race_cd
	   from nsi
	where sourcesystem_cd = nsi.usubjid;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Update subjects with changed demographics in patient_dimension',rowCt,stepCt,'Done') into rtnCd;

	--	insert new subjects into patient_dimension
	
	begin
	insert into i2b2demodata.patient_dimension
    (patient_num,
     sex_cd,
     age_in_years_num,
     race_cd,
     update_date,
     download_date,
     import_date,
     sourcesystem_cd
    )
    select nextval('i2b2demodata.seq_patient_num'),
		   t.sex_cd,
		   t.age_in_years_num,
		   t.race_cd,
		   current_timestamp,
		   current_timestamp,
		   current_timestamp,
		   t.usubjid
    from tm_wz.wt_subject_info t
	where t.usubjid in 
		 (select distinct cd.usubjid from tm_wz.wt_subject_info cd
		  except
		  select distinct pd.sourcesystem_cd from i2b2demodata.patient_dimension pd
		  where pd.sourcesystem_cd like TrialId || ':%');
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Insert new subjects into patient_dimension',rowCt,stepCt,'Done') into rtnCd;
		
	--	delete leaf nodes that will not be reused, if any
	
	 FOR r_delUnusedLeaf in delUnusedLeaf Loop

    --	deletes unused leaf nodes for a trial one at a time

		select tm_cz.i2b2_delete_1_node(r_delUnusedLeaf.c_fullname) into rtnCd;
		stepCt := stepCt + 1;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Deleted unused leaf node: ' || r_delUnusedLeaf.c_fullname,1,stepCt,'Done') into rtnCd;

	END LOOP;
	
	--	bulk insert leaf nodes
	begin
	with ncd as (select t.leaf_node, t.node_name from tm_wz.wt_trial_nodes t)
	update i2b2demodata.concept_dimension
	set name_char=ncd.node_name
	from ncd
	where concept_path = ncd.leaf_node;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Update name_char in concept_dimension for changed names',rowCt,stepCt,'Done') into rtnCd;
	
	begin
	insert into i2b2demodata.concept_dimension
    (concept_cd
	,concept_path
	,name_char
	,update_date
	,download_date
	,import_date
	,sourcesystem_cd
	)
    select nextval('i2b2demodata.concept_id')
	     ,x.leaf_node
		 ,x.node_name
		 ,current_timestamp
		 ,current_timestamp
		 ,current_timestamp
		 ,TrialId
	from (select distinct c.leaf_node
				,c.node_name::text as node_name
		  from tm_wz.wt_trial_nodes c
		  where not exists
			(select 1 from i2b2demodata.concept_dimension x
			where c.leaf_node = x.concept_path)
		 ) x;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Inserted new leaf nodes into I2B2DEMODATA concept_dimension',rowCt,stepCt,'Done') into rtnCd;
	
	--	update i2b2 with name, data_type and xml for leaf nodes
	begin
	with ncd as (select t.leaf_node, t.node_name, t.data_type from tm_wz.wt_trial_nodes t)
	update i2b2metadata.i2b2
	set c_name=ncd.node_name
	   ,c_columndatatype='T'
	   ,c_metadataxml=case when ncd.data_type = 'T'
					  then null
					  else '<?xml version="1.0"?><ValueMetadata><Version>3.02</Version><CreationDateTime>08/14/2008 01:22:59</CreationDateTime><TestID></TestID><TestName></TestName><DataType>PosFloat</DataType><CodeType></CodeType><Loinc></Loinc><Flagstouse></Flagstouse><Oktousevalues>Y</Oktousevalues><MaxStringLength></MaxStringLength><LowofLowValue>0</LowofLowValue><HighofLowValue>0</HighofLowValue><LowofHighValue>100</LowofHighValue>100<HighofHighValue>100</HighofHighValue><LowofToxicValue></LowofToxicValue><HighofToxicValue></HighofToxicValue><EnumValues></EnumValues><CommentsDeterminingExclusion><Com></Com></CommentsDeterminingExclusion><UnitValues><NormalUnits>ratio</NormalUnits><EqualUnits></EqualUnits><ExcludingUnits></ExcludingUnits><ConvertingUnits><Units></Units><MultiplyingFactor></MultiplyingFactor></ConvertingUnits></UnitValues><Analysis><Enums /><Counts /><New /></Analysis></ValueMetadata>'
					  end
	from ncd
	where c_fullname = ncd.leaf_node;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Updated name and data type in i2b2 if changed',rowCt,stepCt,'Done') into rtnCd;
			   
	begin
	insert into i2b2metadata.i2b2
    (c_hlevel
	,c_fullname
	,c_name
	,c_visualattributes
	,c_synonym_cd
	,c_facttablecolumn
	,c_tablename
	,c_columnname
	,c_dimcode
	,c_tooltip
	,update_date
	,download_date
	,import_date
	,sourcesystem_cd
	,c_basecode
	,c_operator
	,c_columndatatype
	,c_comment
	,m_applied_path
	,c_metadataxml
	)
    select (length(c.concept_path) - coalesce(length(replace(c.concept_path, '\','')),0)) / length('\') - 2 + root_level
		  ,c.concept_path
		  ,c.name_char
		  ,'LA'
		  ,'N'
		  ,'CONCEPT_CD'
		  ,'CONCEPT_DIMENSION'
		  ,'CONCEPT_PATH'
		  ,c.concept_path
		  ,c.concept_path
		  ,current_timestamp
		  ,current_timestamp
		  ,current_timestamp
		  ,c.sourcesystem_cd
		  ,c.concept_cd
		  ,'LIKE'
		  ,'T'
		  ,'trial:' || TrialID 
		  ,'@'
		  ,case when t.data_type = 'T' then null
		   else '<?xml version="1.0"?><ValueMetadata><Version>3.02</Version><CreationDateTime>08/14/2008 01:22:59</CreationDateTime><TestID></TestID><TestName></TestName><DataType>PosFloat</DataType><CodeType></CodeType><Loinc></Loinc><Flagstouse></Flagstouse><Oktousevalues>Y</Oktousevalues><MaxStringLength></MaxStringLength><LowofLowValue>0</LowofLowValue><HighofLowValue>0</HighofLowValue><LowofHighValue>100</LowofHighValue>100<HighofHighValue>100</HighofHighValue><LowofToxicValue></LowofToxicValue><HighofToxicValue></HighofToxicValue><EnumValues></EnumValues><CommentsDeterminingExclusion><Com></Com></CommentsDeterminingExclusion><UnitValues><NormalUnits>ratio</NormalUnits><EqualUnits></EqualUnits><ExcludingUnits></ExcludingUnits><ConvertingUnits><Units></Units><MultiplyingFactor></MultiplyingFactor></ConvertingUnits></UnitValues><Analysis><Enums /><Counts /><New /></Analysis></ValueMetadata>'
		   end
    from i2b2demodata.concept_dimension c
		,tm_wz.wt_trial_nodes t
    where c.concept_path = t.leaf_node
	  and not exists
		 (select 1 from i2b2metadata.i2b2 x
		  where c.concept_path = x.c_fullname);
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Inserted leaf nodes into I2B2METADATA i2b2',rowCt,stepCt,'Done') into rtnCd;

	begin
for ul in uploadI2b2
        loop
     update i2b2 n
    SET  --Static XML String
        c_metadataxml =  ('<?xml version="1.0"?><ValueMetadata><Version>3.02</Version>
    <HighofLowValue>0</HighofLowValue><LowofHighValue>100</LowofHighValue>100<HighofHighValue>100</HighofHighValue>
    <CreationDateTime>08/14/2008 01:22:59</CreationDateTime><TestID></TestID><TestName>
    </TestName><DataType>PosFloat</DataType><CodeType></CodeType><Loinc></Loinc><Flagstouse>
    </Flagstouse><Oktousevalues>Y</Oktousevalues><MaxStringLength></MaxStringLength><LowofLowValue>0</LowofLowValue>
    <LowofToxicValue></LowofToxicValue><HighofToxicValue></HighofToxicValue><EnumValues></EnumValues>
    <CommentsDeterminingExclusion><Com></Com></CommentsDeterminingExclusion><UnitValues>
    <NormalUnits>ratio</NormalUnits><EqualUnits></EqualUnits><ExcludingUnits></ExcludingUnits>
    <ConvertingUnits><Units></Units><MultiplyingFactor></MultiplyingFactor></ConvertingUnits>
    </UnitValues><Analysis><Enums /><Counts /><New /></Analysis>   
'||(select xmlelement(name "SeriesMeta",xmlforest(m.display_value as "Value",m.display_unit as "Unit",m.display_label as "DisplayName")) as hi
      from tm_lz.lt_src_display_mapping m where (m.category_cd||m.display_label)=(ul.category_cd||ul.display_label))||
                '</ValueMetadata>') where n.c_fullname in  
				(select leaf_node from wt_trial_nodes where (((category_cd||'+'||replace(data_label,'PCT','%'))||node_name)=(ul.category_cd||ul.display_label) 
				or (category_cd||node_name)=(ul.category_cd||ul.display_label)) and leaf_node=n.c_fullname);
                
                end loop;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Updated I2B2 for metadataXML',rowCt,stepCt,'Done') into rtnCd;

				
	--	delete from observation_fact all concept_cds for trial that are clinical data, exclude concept_cds from biomarker data
	
	begin
	delete from i2b2demodata.observation_fact f
	where f.sourcesystem_cd = TrialId
	  and f.concept_cd not in
		 (select distinct concept_code as concept_cd from deapp.de_subject_sample_mapping
		  where trial_name = TrialId
		    and concept_code is not null
		  union
		  select distinct platform_cd as concept_cd from deapp.de_subject_sample_mapping
		  where trial_name = TrialId
		    and platform_cd is not null
		  union
		  select distinct sample_type_cd as concept_cd from deapp.de_subject_sample_mapping
		  where trial_name = TrialId
		    and sample_type_cd is not null
		  union
		  select distinct tissue_type_cd as concept_cd from deapp.de_subject_sample_mapping
		  where trial_name = TrialId
		    and tissue_type_cd is not null
		  union
		  select distinct timepoint_cd as concept_cd from deapp.de_subject_sample_mapping
		  where trial_name = TrialId
		    and timepoint_cd is not null
		  union
		  select distinct concept_cd as concept_cd from deapp.de_subject_snp_dataset
		  where trial_name = TrialId
		    and concept_cd is not null);
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete clinical data for study from observation_fact',rowCt,stepCt,'Done') into rtnCd;  
	
    --Insert into observation_fact
	
	begin
	insert into i2b2demodata.observation_fact
	(encounter_num,
     patient_num,
     concept_cd,
     start_date,
     modifier_cd,
     valtype_cd,
     tval_char,
     nval_num,
     units_cd,
     sourcesystem_cd,
     import_date,
     valueflag_cd,
     provider_id,
     location_cd,
     instance_num
	)
	select distinct c.patient_num,
		   c.patient_num,
		   i.c_basecode,
		   coalesce(a.date_timestamp, 'infinity'),
		   coalesce(a.modifier_cd, '@'),
		   a.data_type,
		   case when a.data_type = 'T' then a.data_value
				else 'E'  --Stands for Equals for numeric types
				end,
		   case when a.data_type = 'N' then a.data_value::numeric
				else null --Null for text types
				end,
                   a.units_cd,
		   a.study_id, 
		   current_timestamp, 
		   '@',
		   '@',
		   '@',
                   1
	from tm_wz.wrk_clinical_data a
		,i2b2demodata.patient_dimension c
		,tm_wz.wt_trial_nodes t
		,i2b2metadata.i2b2 i
	where a.usubjid = c.sourcesystem_cd
	  and coalesce(a.category_cd,'@') = coalesce(t.category_cd,'@')
	  and coalesce(a.data_label,'**NULL**') = coalesce(t.data_label,'**NULL**')
	  and coalesce(a.visit_name,'**NULL**') = coalesce(t.visit_name,'**NULL**')
	  and case when a.data_type = 'T' then a.data_value else '**NULL**' end = coalesce(t.data_value,'**NULL**')
	  and t.leaf_node = i.c_fullname
	  and not exists		-- don't insert if lower level node exists
		 (select 1 from tm_wz.wt_trial_nodes x
		  where x.leaf_node like t.leaf_node || '%_' escape '`')
	  and a.data_value is not null;
	get diagnostics rowCt := ROW_COUNT; 
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Insert trial into I2B2DEMODATA observation_fact',rowCt,stepCt,'Done') into rtnCd;

	--	update c_visualattributes for all nodes in study, done to pick up node that changed c_columndatatype
	
	begin
	with upd as (select p.c_fullname, count(*) as nbr_children 
				 from i2b2metadata.i2b2 p
					 ,i2b2metadata.i2b2 c
				 where p.c_fullname like topNode || '%' escape '`'
				   and c.c_fullname like p.c_fullname || '%' escape '`'
				 group by p.c_fullname)
	update i2b2metadata.i2b2 b
	set c_visualattributes=case when upd.nbr_children = 1 
								then 'L' || substr(b.c_visualattributes,2,2)
								else 'F' || substr(b.c_visualattributes,2,1) ||
									case when upd.c_fullname = topNode
										then case when highlight_study = 'Y' then 'J' else 'S' end 
									 else substr(b.c_visualattributes,3,1) end
								end
		,c_columndatatype=case when upd.nbr_children > 1 then 'T' else b.c_columndatatype end
	from upd
	where b.c_fullname = upd.c_fullname
	  and b.c_fullname in
		 (select x.c_fullname from i2b2metadata.i2b2 x
		  where x.c_fullname like topNode || '%' escape '`');
  	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Set c_visualattributes in i2b2',rowCt,stepCt,'Done') into rtnCd;

	-- final procs
  
	select tm_cz.i2b2_fill_in_tree(TrialId, topNode, jobID) into rtnCd;
	
	select tm_cz.i2b2_create_concept_counts(topNode, jobID) into rtnCd;
	
	--	delete each node that is hidden after create concept counts
	
	 FOR r_delNodes in delNodes Loop

    --	deletes hidden nodes for a trial one at a time

		select tm_cz.i2b2_delete_1_node(r_delNodes.c_fullname) into rtnCd;
		stepCt := stepCt + 1;
		tText := 'Deleted node: ' || r_delNodes.c_fullname;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,tText,rowCt,stepCt,'Done') into rtnCd;

	END LOOP;  	

	select tm_cz.i2b2_create_security_for_trial(TrialId, secureStudy, jobID) into rtnCd;
	select tm_cz.i2b2_load_security_data(jobID) into rtnCd;
	
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'End i2b2_load_clinical_data',0,stepCt,'Done') into rtnCd;
	
	---Cleanup OVERALL JOB if this proc is being run standalone
	IF newJobFlag = 1
	THEN
		select tm_cz.cz_end_audit (jobID, 'SUCCESS') into rtnCd;
	END IF;

	return 1;
END;

$$;


ALTER FUNCTION tm_cz.i2b2_load_clinical_data(trial_id character varying, top_node character varying, secure_study character varying, highlight_study character varying, currentjobid numeric) OWNER TO tm_cz;

--
-- Name: i2b2_load_clinical_inc_data(character varying, character varying, character varying, character varying, numeric); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_load_clinical_inc_data(trial_id character varying, top_node character varying, secure_study character varying DEFAULT 'N'::character varying, highlight_study character varying DEFAULT 'N'::character varying, currentjobid numeric DEFAULT '-1'::integer) RETURNS numeric
    LANGUAGE plpgsql SECURITY DEFINER
    AS $$
/*
This procedure is created for incremental data load for clinical data
*/
Declare
 
	--Audit variables
	newJobFlag		integer;
	databaseName 	VARCHAR(100);
	procedureName 	VARCHAR(100);
	jobID 			numeric(18,0);
	stepCt 			numeric(18,0);
	rowCt			numeric(18,0);
	errorNumber		character varying;
	errorMessage	character varying;
	rtnCd			integer;
	
	topNode			varchar(2000);
	topLevel		numeric(10,0);
	root_node		varchar(2000);
	root_level		integer;
	study_name		varchar(2000);
	TrialID			varchar(100);
	secureStudy		varchar(200);
	etlDate			timestamp;
	tPath			varchar(2000);
	pCount			integer;
	pExists			integer;
	rtnCode			integer;
	tText			varchar(2000);
  
	addNodes CURSOR is
	select DISTINCT leaf_node, node_name
	from  tm_wz.wt_trial_nodes a;
   
	--	cursor to define the path for delete_one_node  this will delete any nodes that are hidden after i2b2_create_concept_counts

	delNodes CURSOR is
	select distinct c_fullname 
	from  i2b2metadata.i2b2
	where c_fullname like topNode || '%' escape '`'
      and substr(c_visualattributes,2,1) = 'H';
	  
	--	cursor to determine if any leaf nodes exist in i2b2 that are not used in this reload (node changes from text to numeric or numeric to text)
	  
	delUnusedLeaf cursor is
	select l.c_fullname
	from i2b2metadata.i2b2 l
	where l.c_visualattributes like 'L%'
	  and l.c_fullname like topNode || '%' escape '`'
	  and l.c_fullname not in
		 (select t.leaf_node 
		  from tm_wz.wt_trial_nodes t
		  union
		  select m.c_fullname
		  from deapp.de_subject_sample_mapping sm
			  ,i2b2metadata.i2b2 m
		  where sm.trial_name = TrialId
		    and sm.concept_code = m.c_basecode
			and m.c_visualattributes like 'L%');
			
	-- added by Cognizant for requirement 3 and 4 under #1
    uploadI2b2 cursor is
    select category_cd,display_value,display_label,display_unit from
    tm_lz.lt_src_display_mapping group by category_cd,display_value,display_label,display_unit;
BEGIN
  
	TrialID := upper(trial_id);
	secureStudy := upper(secure_study);
	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;
	
	

	databaseName := 'TM_CZ';
	procedureName := 'I2B2_LOAD_CLINICAL_INC_DATA';
	
	
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Start i2b2_load_clinical_inc_data ',0,stepCt,'Done') into rtnCd;

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it

	IF(jobID IS NULL or jobID < 1)
	THEN
		newJobFlag := 1; -- True
		select tm_cz.cz_start_audit (procedureName, databaseName) into jobID;
	END IF;
    	
	stepCt := 0;
	stepCt := stepCt + 1;
	tText := 'Start i2b2_load_clinical_inc_data for ' || TrialId;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,tText,0,stepCt,'Done') into rtnCd;
	
	if (secureStudy not in ('Y','N') ) then
		secureStudy := 'Y';
	end if;
	
	topNode := REGEXP_REPLACE('\' || top_node || '\','(\\){2,}', '\', 'g');
	
	--	figure out how many nodes (folders) are at study name and above
	--	\Public Studies\Clinical Studies\Pancreatic_Cancer_Smith_GSE22780\: topLevel = 4, so there are 3 nodes
	--	\Public Studies\GSE12345\: topLevel = 3, so there are 2 nodes
	
	select length(topNode)-length(replace(topNode,'\','')) into topLevel;
	
	if topLevel < 3 then
		stepCt := stepCt + 1;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Path specified in top_node must contain at least 2 nodes',0,stepCt,'Done') into rtnCd;
		select tm_cz.cz_error_handler (jobID, procedureName, '-1', 'Application raised error') into rtnCd;
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end if;

	-- inc-change delete only data to be replaced

	--	delete any existing data from lz_src_clinical_data and load new data
	begin
	delete from tm_lz.lz_src_clinical_data
	where (study_id||subject_id||category_cd) in( select (study_id||subject_id||category_cd ) from lt_src_clinical_data);
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	get diagnostics rowCt := ROW_COUNT;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete existing data from lz_src_clinical_data',rowCt,stepCt,'Done') into rtnCd;
	
	begin
	insert into tm_lz.lz_src_clinical_data
	(study_id
	,site_id
	,subject_id
	,visit_name
	,data_label
        ,modifier_cd
	,data_value
        ,units_cd
        ,date_timestamp
	,category_cd
	,etl_job_id
	,etl_date
	,ctrl_vocab_code)
	select study_id
		  ,site_id
		  ,subject_id
		  ,visit_name
		  ,data_label
                  ,modifier_cd
		  ,data_value
                  ,units_cd
                  ,date_timestamp
		  ,category_cd
		  ,jobId
		  ,etlDate
		  ,ctrl_vocab_code
	from tm_lz.lt_src_clinical_data;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	get diagnostics rowCt := ROW_COUNT;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Insert data into lz_src_clinical_data',rowCt,stepCt,'Done') into rtnCd;
		
	--	truncate tm_wz.wrk_clinical_data and load data from external file
	
	execute ('truncate table tm_wz.wrk_clinical_data');
	
	--	insert data from lt_src_clinical_data to tm_wz.wrk_clinical_data
	
	begin
	insert into tm_wz.wrk_clinical_data
	(study_id
	,site_id
	,subject_id
	,visit_name
	,data_label
        ,modifier_cd
	,data_value
        ,units_cd
        ,date_timestamp
	,category_cd
	,ctrl_vocab_code
	)
	select study_id
		  ,site_id
		  ,subject_id
		  ,visit_name
		  ,data_label
                  ,modifier_cd
		  ,data_value
                  ,units_cd
                  ,date_timestamp
		  ,category_cd
		  ,ctrl_vocab_code
	from tm_lz.lt_src_clinical_data;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	get diagnostics rowCt := ROW_COUNT;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Load lt_src_clinical_data to work table',rowCt,stepCt,'Done') into rtnCd;

	-- Get root_node from topNode
  
	select tm_cz.parse_nth_value(topNode, 2, '\') into root_node;
	
	select count(*) into pExists
	from i2b2metadata.table_access
	where c_name = root_node;
	
	select count(*) into pCount
	from i2b2metadata.i2b2
	where c_name = root_node;
	
	if pExists = 0 or pCount = 0 then
		select tm_cz.i2b2_add_root_node(root_node, jobId) into rtnCd;
	end if;
	
	select c_hlevel into root_level
	from i2b2metadata.table_access
	where c_name = root_node;
	
	-- Get study name from topNode
  
	select tm_cz.parse_nth_value(topNode, topLevel, '\') into study_name;
	
	--	Add any upper level nodes as needed
	
	tPath := REGEXP_REPLACE(replace(top_node,study_name,''),'(\\){2,}', '\', 'g');
	select length(tPath) - length(replace(tPath,'\','')) into pCount;

	if pCount > 2 then
		stepCt := stepCt + 1;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Adding upper-level nodes',0,stepCt,'Done') into rtnCd;
		select tm_cz.i2b2_fill_in_tree(null, tPath, jobId) into rtnCd;
	end if;

	select count(*) into pExists
	from i2b2metadata.i2b2
	where c_fullname = topNode;
	
	--	add top node for study
	
	if pExists = 0 then
		select tm_cz.i2b2_add_node(TrialId, topNode, study_name, jobId) into rtnCd;
	end if;
  
	--	Set data_type, category_path, and usubjid 
  
	update tm_wz.wrk_clinical_data
	set data_type = 'T'
	   ,category_path = replace(replace(category_cd,'_',' '),'+','\')
	   ,usubjid = REGEXP_REPLACE(TrialID || ':' || coalesce(site_id,'') || ':' || subject_id,
                   '(::){1,}', ':', 'g'); 
	 get diagnostics rowCt := ROW_COUNT;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Set columns in tm_wz.wrk_clinical_data',rowCt,stepCt,'Done') into rtnCd;

	--	Delete rows where data_value is null
  
	begin
	delete from tm_wz.wrk_clinical_data
	where data_value is null;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	get diagnostics rowCt := ROW_COUNT;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete null data_values in tm_wz.wrk_clinical_data',rowCt,stepCt,'Done') into rtnCd;
	
	--Remove Invalid pipes in the data values.
	--RULE: If Pipe is last or first, delete it
	--If it is in the middle replace with a dash

	begin
	update tm_wz.wrk_clinical_data
	set data_value = replace(trim('|' from data_value), '|', '-')
	where data_value like '%|%';
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	get diagnostics rowCt := ROW_COUNT;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Remove pipes in data_value',rowCt,stepCt,'Done') into rtnCd;
 
	--Remove invalid Parens in the data
	--They have appeared as empty pairs or only single ones.
  
	begin
	update tm_wz.wrk_clinical_data
	set data_value = replace(data_value,'(', '')
	where data_value like '%()%'
	   or data_value like '%( )%'
	   or (data_value like '%(%' and data_value NOT like '%)%');
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Remove empty parentheses 1',rowCt,stepCt,'Done') into rtnCd;
	
	begin
	update tm_wz.wrk_clinical_data
	set data_value = replace(data_value,')', '')
	where data_value like '%()%'
	   or data_value like '%( )%'
	   or (data_value like '%)%' and data_value NOT like '%(%');
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Remove empty parentheses 2',rowCt,stepCt,'Done') into rtnCd;

	--Replace the Pipes with Commas in the data_label column
	begin
	update tm_wz.wrk_clinical_data
    set data_label = replace (data_label, '|', ',')
    where data_label like '%|%';
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Replace pipes with comma in data_label',rowCt,stepCt,'Done') into rtnCd;

	--	set visit_name to null when there's only a single visit_name for the catgory
	
	begin
	update tm_wz.wrk_clinical_data tpm
	set visit_name=null
	where (tpm.category_cd) in
		  (select x.category_cd
		   from tm_wz.wrk_clinical_data x
		   group by x.category_cd
		   having count(distinct upper(x.visit_name)) = 1);
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Set single visit_name to null',rowCt,stepCt,'Done') into rtnCd;
	
	--	set data_label to null when it duplicates the last part of the category_path
	--	Remove data_label from last part of category_path when they are the same

	begin
	update tm_wz.wrk_clinical_data tpm
	--set data_label = null
	set category_path=substr(tpm.category_path,1,tm_cz.instr(tpm.category_path,'\',-2,1)-1)
	   ,category_cd=substr(tpm.category_cd,1,tm_cz.instr(tpm.category_cd,'+',-2,1)-1)
	where tm_cz.instr(tpm.category_path,'\',-1,1) > 0 and (tpm.category_cd, tpm.data_label) in
		  (select distinct t.category_cd
				 ,t.data_label
		   from tm_wz.wrk_clinical_data t
		   where upper(substr(t.category_path,tm_cz.instr(t.category_path,'\',-1,1)+1,length(t.category_path)-tm_cz.instr(t.category_path,'\',-1,1))) 
			     = upper(t.data_label)
		     and t.data_label is not null)
	  and tpm.data_label is not null;
        update tm_wz.wrk_clinical_data tpm
        set data_label = null
        where tm_cz.instr(tpm.category_path,'\',-1,1) = 0 and (tpm.category_cd, tpm.data_label) in
                  (select distinct t.category_cd
                                 ,t.data_label
                   from tm_wz.wrk_clinical_data t
                   where upper(substr(t.category_path,tm_cz.instr(t.category_path,'\',-1,1)+1,length(t.category_path)-tm_cz.instr(t.category_path,'\',-1,1)))
                             = upper(t.data_label)
                     and t.data_label is not null)
          and tpm.data_label is not null;
        get diagnostics rowCt := ROW_COUNT;
        exception
        when others then
                errorNumber := SQLSTATE;
                errorMessage := SQLERRM;
                --Handle errors.
                select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
                --End Proc
                select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
                return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Set data_label to null when found in category_path',rowCt,stepCt,'Done') into rtnCd;

	--	set visit_name to null if same as data_label
	
	begin
	update tm_wz.wrk_clinical_data t
	set visit_name=null
	where (t.category_cd, t.visit_name, t.data_label) in
	      (select distinct tpm.category_cd
				 ,tpm.visit_name
				 ,tpm.data_label
		  from tm_wz.wrk_clinical_data tpm
		  where tpm.visit_name = tpm.data_label);
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Set visit_name to null when found in data_label',rowCt,stepCt,'Done') into rtnCd;

	--	set visit_name to null if same as data_value
	
	begin
	update tm_wz.wrk_clinical_data t
	set visit_name=null
	where (t.category_cd, t.visit_name, t.data_value) in
	      (select distinct tpm.category_cd
				 ,tpm.visit_name
				 ,tpm.data_value
		  from tm_wz.wrk_clinical_data tpm
		  where tpm.visit_name = tpm.data_value);
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Set visit_name to null when found in data_value',rowCt,stepCt,'Done') into rtnCd;

	--	set visit_name to null if only DATALABEL in category_cd
	
	begin
	update tm_wz.wrk_clinical_data t
	set visit_name=null
	where t.category_cd like '%DATALABEL%'
	  and t.category_cd not like '%VISITNAME%';
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Set visit_name to null when only DATALABE in category_cd',rowCt,stepCt,'Done') into rtnCd;
	
	--	change any % to Pct and & and + to ' and ' and _ to space in data_label only
	
	begin
	update tm_wz.wrk_clinical_data
	set data_label=replace(replace(replace(replace(data_label,'%',' Pct'),'&',' and '),'+',' and '),'_',' ')
	   ,data_value=replace(replace(replace(data_value,'%',' Pct'),'&',' and '),'+',' and ')
	   ,category_cd=replace(replace(category_cd,'%',' Pct'),'&',' and ')
	   ,category_path=replace(replace(category_path,'%',' Pct'),'&',' and ');
	   exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;

  --Trim trailing and leadling spaces as well as remove any double spaces, remove space from before comma, remove trailing comma

	begin
	update tm_wz.wrk_clinical_data
	set data_label  = trim(trailing ',' from trim(replace(replace(data_label,'  ', ' '),' ,',','))),
		data_value  = trim(trailing ',' from trim(replace(replace(data_value,'  ', ' '),' ,',','))),
--		sample_type = trim(trailing ',' from trim(replace(replace(sample_type,'  ', ' '),' ,',','))),
		visit_name  = trim(trailing ',' from trim(replace(replace(visit_name,'  ', ' '),' ,',',')));
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Remove leading, trailing, double spaces',rowCt,stepCt,'Done') into rtnCd;

    --1. DETERMINE THE DATA_TYPES OF THE FIELDS
	--	replaced cursor with update, used temp table to store category_cd/data_label because correlated subquery ran too long
	
	execute ('truncate table tm_wz.wt_num_data_types');

	begin
	insert into tm_wz.wt_num_data_types
	(category_cd
	,data_label
	,visit_name
	)
    select category_cd,
           data_label,
           visit_name
    from tm_wz.wrk_clinical_data
    where data_value is not null
    group by category_cd
	        ,data_label
            ,visit_name
      having sum(tm_cz.is_numeric(data_value)) = 0;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Insert numeric data into WZ wt_num_data_types',rowCt,stepCt,'Done') into rtnCd;

	--	Check if any duplicate records of key columns (site_id, subject_id, visit_name, data_label, category_cd) for numeric data
	--	exist.  Raise error if yes
	
	execute ('truncate table tm_wz.wt_clinical_data_dups');
	
	begin
	insert into tm_wz.wt_clinical_data_dups
	(site_id
	,subject_id
	,visit_name
	,data_label
	,category_cd
        ,modifier_cd)
	select w.site_id, w.subject_id, w.visit_name, w.data_label, w.category_cd, w.modifier_cd
	from tm_wz.wrk_clinical_data w
	where exists
		 (select 1 from tm_wz.wt_num_data_types t
		 where coalesce(w.category_cd,'@') = coalesce(t.category_cd,'@')
		   and coalesce(w.data_label,'@') = coalesce(t.data_label,'@')
		   and coalesce(w.visit_name,'@') = coalesce(t.visit_name,'@')
		  )
	group by w.site_id, w.subject_id, w.visit_name, w.data_label, w.category_cd, w.modifier_cd
	having count(*) > 1;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Check for duplicate key columns',rowCt,stepCt,'Done') into rtnCd;
			  
	if rowCt > 0 then
		stepCt := stepCt + 1;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Duplicate values found in key columns',0,stepCt,'Done') into rtnCd;
		select tm_cz.cz_error_handler (jobID, procedureName, '-1', 'Application raised error') into rtnCd;
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end if;
	
	--	check for multiple visit_names for category_cd, data_label, data_value
	
     select max(case when x.null_ct > 0 and x.non_null_ct > 0
					 then 1 else 0 end) into pCount
      from (select category_cd, data_label, data_value
				  ,sum(case when visit_name is null then 1 else 0 end) as null_ct
				  ,sum(case when visit_name is null then 0 else 1 end) as non_null_ct
			from tm_lz.lt_src_clinical_data
			where (category_cd like '%VISITNAME%' or
				   category_cd not like '%DATALABEL%')
			group by category_cd, data_label, data_value) x;
	get diagnostics rowCt := ROW_COUNT;  
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Check for multiple visit_names for category/label/value ',rowCt,stepCt,'Done') into rtnCd;
			  
	if pCount > 0 then
		stepCt := stepCt + 1;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Multiple visit names for category/label/value',0,stepCt,'Done') into rtnCd;
		select tm_cz.cz_error_handler (jobID, procedureName, '-1', 'Application raised error') into rtnCd;
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end if;
		
	begin
	update tm_wz.wrk_clinical_data t
	set data_type='N'
	where exists
	     (select 1 from tm_wz.wt_num_data_types x
	      where coalesce(t.category_cd,'@') = coalesce(x.category_cd,'@')
			and coalesce(t.data_label,'**NULL**') = coalesce(x.data_label,'**NULL**')
			and coalesce(t.visit_name,'**NULL**') = coalesce(x.visit_name,'**NULL**')
		  );
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Updated data_type flag for numeric data_types',rowCt,stepCt,'Done') into rtnCd;

	-- Build all needed leaf nodes in one pass for both numeric and text nodes
 
	execute ('truncate table tm_wz.wt_trial_nodes');
	
	begin
	insert into tm_wz.wt_trial_nodes
	(leaf_node
	,category_cd
	,visit_name
	,data_label
	,data_value
	,data_type
	)
    select DISTINCT 
    Case 
	--	Text data_type (default node)
	When a.data_type = 'T'
	     then case when a.category_path like '%DATALABEL%' and a.category_path like '%VISITNAME%'
		      then regexp_replace(topNode || replace(replace(coalesce(a.category_path,''),'DATALABEL',coalesce(a.data_label,'')),'VISITNAME',coalesce(a.visit_name,'')) || '\' || coalesce(a.data_value,'') || '\','(\\){2,}', '\', 'g')
			  when a.category_path like '%DATALABEL%'
			  then regexp_replace(topNode || replace(coalesce(a.category_path,''),'DATALABEL',coalesce(a.data_label,'')) || '\' || coalesce(a.data_value,'') || '\','(\\){2,}', '\', 'g')
			  else REGEXP_REPLACE(topNode || coalesce(a.category_path,'') || 
                   '\'  || coalesce(a.data_label,'') || '\' || coalesce(a.data_value,'') || '\' || coalesce(a.visit_name,'') || '\',
                   '(\\){2,}', '\', 'g') 
			  end
	--	else is numeric data_type and default_node
	else case when a.category_path like '%DATALABEL%' and a.category_path like '%VISITNAME%'
		      then regexp_replace(topNode || replace(replace(coalesce(a.category_path,''),'DATALABEL',coalesce(a.data_label,'')),'VISITNAME',coalesce(a.visit_name,'')) || '\','(\\){2,}', '\', 'g')
			  when a.category_path like '%DATALABEL%'
			  then regexp_replace(topNode || replace(coalesce(a.category_path,''),'DATALABEL',coalesce(a.data_label,'')) || '\','(\\){2,}', '\', 'g')
			  else REGEXP_REPLACE(topNode || coalesce(a.category_path,'') || 
                   '\'  || coalesce(a.data_label,'') || '\' || coalesce(a.visit_name,'') || '\',
                   '(\\){2,}', '\', 'g')
			  end
	end as leaf_node,
    a.category_cd,
    a.visit_name,
	a.data_label,
	case when a.data_type = 'T' then a.data_value else null end as data_value
    ,a.data_type
	from  tm_wz.wrk_clinical_data a;
	get diagnostics rowCt := ROW_COUNT; 
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Create leaf nodes for trial',rowCt,stepCt,'Done') into rtnCd;

	--	set node_name
	
	begin
	update tm_wz.wt_trial_nodes
	set node_name=tm_cz.parse_nth_value(leaf_node,length(leaf_node)-length(replace(leaf_node,'\','')),'\');
	get diagnostics rowCt := ROW_COUNT; 
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Updated node name for leaf nodes',rowCt,stepCt,'Done') into rtnCd;

	--	insert subjects into patient_dimension if needed
	
	execute ('truncate table tm_wz.wt_subject_info');

	begin
	insert into tm_wz.wt_subject_info
	(usubjid,
     age_in_years_num,
     sex_cd,
     race_cd
    )
	select a.usubjid,
	      max(case when upper(a.data_label) = 'AGE'
				   then case when tm_cz.is_numeric(a.data_value) = 1 then null else round(a.data_value::numeric) end
		           when upper(a.data_label) like '%(AGE)' 
				   then case when tm_cz.is_numeric(a.data_value) = 1 then null else round(a.data_value::numeric) end
				   else null end) as age,
		  max(case when upper(a.data_label) = 'SEX' then a.data_value
		           when upper(a.data_label) like '%(SEX)' then a.data_value
				   when upper(a.data_label) = 'GENDER' then a.data_value
				   else null end) as sex,
		  max(case when upper(a.data_label) = 'RACE' then a.data_value
		           when upper(a.data_label) like '%(RACE)' then a.data_value
				   else null end) as race
	from tm_wz.wrk_clinical_data a
	group by a.usubjid;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Insert subject information into temp table',rowCt,stepCt,'Done') into rtnCd;

	-- inc-change: do not drop existing subjects

        /*
	--	Delete dropped subjects from patient_dimension if they do not exist in de_subject_sample_mapping
	
	begin
	delete from i2b2demodata.patient_dimension
	where sourcesystem_cd in
		 (select distinct pd.sourcesystem_cd from i2b2demodata.patient_dimension pd
		  where pd.sourcesystem_cd like TrialId || ':%'
		  except 
		  select distinct cd.usubjid from tm_wz.wrk_clinical_data cd)
	  and patient_num not in
		  (select distinct sm.patient_id from deapp.de_subject_sample_mapping sm);
	get diagnostics rowCt := ROW_COUNT; 
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete dropped subjects from patient_dimension',rowCt,stepCt,'Done') into rtnCd;
        */

	--	update patients with changed information
	begin
	with nsi as (select t.usubjid, t.sex_cd, t.age_in_years_num, t.race_cd from tm_wz.wt_subject_info t) 
	update i2b2demodata.patient_dimension
	set sex_cd=nsi.sex_cd
	   ,age_in_years_num=nsi.age_in_years_num
	   ,race_cd=nsi.race_cd
	   from nsi
	where sourcesystem_cd = nsi.usubjid;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Update subjects with changed demographics in patient_dimension',rowCt,stepCt,'Done') into rtnCd;

	--	insert new subjects into patient_dimension
	
	begin
	insert into i2b2demodata.patient_dimension
    (patient_num,
     sex_cd,
     age_in_years_num,
     race_cd,
     update_date,
     download_date,
     import_date,
     sourcesystem_cd
    )
    select nextval('i2b2demodata.seq_patient_num'),
		   t.sex_cd,
		   t.age_in_years_num,
		   t.race_cd,
		   current_timestamp,
		   current_timestamp,
		   current_timestamp,
		   t.usubjid
    from tm_wz.wt_subject_info t
	where t.usubjid in 
		 (select distinct cd.usubjid from tm_wz.wt_subject_info cd
		  except
		  select distinct pd.sourcesystem_cd from i2b2demodata.patient_dimension pd
		  where pd.sourcesystem_cd like TrialId || ':%');
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Insert new subjects into patient_dimension',rowCt,stepCt,'Done') into rtnCd;

	-- inc-change: keep all existing leaf nodes

        /*
	--	delete leaf nodes that will not be reused, if any
	
	 FOR r_delUnusedLeaf in delUnusedLeaf Loop

    --	deletes unused leaf nodes for a trial one at a time

		select tm_cz.i2b2_delete_1_node(r_delUnusedLeaf.c_fullname) into rtnCd;
		stepCt := stepCt + 1;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Deleted unused leaf node: ' || r_delUnusedLeaf.c_fullname,1,stepCt,'Done') into rtnCd;

	END LOOP;
        */	
	--	bulk insert leaf nodes
	begin
	with ncd as (select t.leaf_node, t.node_name from tm_wz.wt_trial_nodes t)
	update i2b2demodata.concept_dimension
	set name_char=ncd.node_name
	from ncd
	where concept_path = ncd.leaf_node;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Update name_char in concept_dimension for changed names',rowCt,stepCt,'Done') into rtnCd;
	
	begin
	insert into i2b2demodata.concept_dimension
    (concept_cd
	,concept_path
	,name_char
	,update_date
	,download_date
	,import_date
	,sourcesystem_cd
	)
    select nextval('i2b2demodata.concept_id')
	     ,x.leaf_node
		 ,x.node_name
		 ,current_timestamp
		 ,current_timestamp
		 ,current_timestamp
		 ,TrialId
	from (select distinct c.leaf_node
				,c.node_name::text as node_name
		  from tm_wz.wt_trial_nodes c
		  where not exists
			(select 1 from i2b2demodata.concept_dimension x
			where c.leaf_node = x.concept_path)
		 ) x;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Inserted new leaf nodes into I2B2DEMODATA concept_dimension',rowCt,stepCt,'Done') into rtnCd;
	
	--	update i2b2 with name, data_type and xml for leaf nodes
	begin
	with ncd as (select t.leaf_node, t.node_name, t.data_type from tm_wz.wt_trial_nodes t)
	update i2b2metadata.i2b2
	set c_name=ncd.node_name
	   ,c_columndatatype='T'
	   ,c_metadataxml=case when ncd.data_type = 'T'
					  then null
					  else '<?xml version="1.0"?><ValueMetadata><Version>3.02</Version><CreationDateTime>08/14/2008 01:22:59</CreationDateTime><TestID></TestID><TestName></TestName><DataType>PosFloat</DataType><CodeType></CodeType><Loinc></Loinc><Flagstouse></Flagstouse><Oktousevalues>Y</Oktousevalues><MaxStringLength></MaxStringLength><LowofLowValue>0</LowofLowValue><HighofLowValue>0</HighofLowValue><LowofHighValue>100</LowofHighValue>100<HighofHighValue>100</HighofHighValue><LowofToxicValue></LowofToxicValue><HighofToxicValue></HighofToxicValue><EnumValues></EnumValues><CommentsDeterminingExclusion><Com></Com></CommentsDeterminingExclusion><UnitValues><NormalUnits>ratio</NormalUnits><EqualUnits></EqualUnits><ExcludingUnits></ExcludingUnits><ConvertingUnits><Units></Units><MultiplyingFactor></MultiplyingFactor></ConvertingUnits></UnitValues><Analysis><Enums /><Counts /><New /></Analysis></ValueMetadata>'
					  end
	from ncd
	where c_fullname = ncd.leaf_node;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Updated name and data type in i2b2 if changed',rowCt,stepCt,'Done') into rtnCd;
			   
	begin
	insert into i2b2metadata.i2b2
    (c_hlevel
	,c_fullname
	,c_name
	,c_visualattributes
	,c_synonym_cd
	,c_facttablecolumn
	,c_tablename
	,c_columnname
	,c_dimcode
	,c_tooltip
	,update_date
	,download_date
	,import_date
	,sourcesystem_cd
	,c_basecode
	,c_operator
	,c_columndatatype
	,c_comment
	,m_applied_path
	,c_metadataxml
	)
    select (length(c.concept_path) - coalesce(length(replace(c.concept_path, '\','')),0)) / length('\') - 2 + root_level
		  ,c.concept_path
		  ,c.name_char
		  ,'LA'
		  ,'N'
		  ,'CONCEPT_CD'
		  ,'CONCEPT_DIMENSION'
		  ,'CONCEPT_PATH'
		  ,c.concept_path
		  ,c.concept_path
		  ,current_timestamp
		  ,current_timestamp
		  ,current_timestamp
		  ,c.sourcesystem_cd
		  ,c.concept_cd
		  ,'LIKE'
		  ,'T'
		  ,'trial:' || TrialID 
		  ,'@'
		  ,case when t.data_type = 'T' then null
		   else '<?xml version="1.0"?><ValueMetadata><Version>3.02</Version><CreationDateTime>08/14/2008 01:22:59</CreationDateTime><TestID></TestID><TestName></TestName><DataType>PosFloat</DataType><CodeType></CodeType><Loinc></Loinc><Flagstouse></Flagstouse><Oktousevalues>Y</Oktousevalues><MaxStringLength></MaxStringLength><LowofLowValue>0</LowofLowValue><HighofLowValue>0</HighofLowValue><LowofHighValue>100</LowofHighValue>100<HighofHighValue>100</HighofHighValue><LowofToxicValue></LowofToxicValue><HighofToxicValue></HighofToxicValue><EnumValues></EnumValues><CommentsDeterminingExclusion><Com></Com></CommentsDeterminingExclusion><UnitValues><NormalUnits>ratio</NormalUnits><EqualUnits></EqualUnits><ExcludingUnits></ExcludingUnits><ConvertingUnits><Units></Units><MultiplyingFactor></MultiplyingFactor></ConvertingUnits></UnitValues><Analysis><Enums /><Counts /><New /></Analysis></ValueMetadata>'
		   end
    from i2b2demodata.concept_dimension c
		,tm_wz.wt_trial_nodes t
    where c.concept_path = t.leaf_node
	  and not exists
		 (select 1 from i2b2metadata.i2b2 x
		  where c.concept_path = x.c_fullname);
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Inserted leaf nodes into I2B2METADATA i2b2',rowCt,stepCt,'Done') into rtnCd;

	begin
for ul in uploadI2b2
        loop
     update i2b2 n
    SET  --Static XML String
        c_metadataxml =  ('<?xml version="1.0"?><ValueMetadata><Version>3.02</Version>
    <HighofLowValue>0</HighofLowValue><LowofHighValue>100</LowofHighValue>100<HighofHighValue>100</HighofHighValue>
    <CreationDateTime>08/14/2008 01:22:59</CreationDateTime><TestID></TestID><TestName>
    </TestName><DataType>PosFloat</DataType><CodeType></CodeType><Loinc></Loinc><Flagstouse>
    </Flagstouse><Oktousevalues>Y</Oktousevalues><MaxStringLength></MaxStringLength><LowofLowValue>0</LowofLowValue>
    <LowofToxicValue></LowofToxicValue><HighofToxicValue></HighofToxicValue><EnumValues></EnumValues>
    <CommentsDeterminingExclusion><Com></Com></CommentsDeterminingExclusion><UnitValues>
    <NormalUnits>ratio</NormalUnits><EqualUnits></EqualUnits><ExcludingUnits></ExcludingUnits>
    <ConvertingUnits><Units></Units><MultiplyingFactor></MultiplyingFactor></ConvertingUnits>
    </UnitValues><Analysis><Enums /><Counts /><New /></Analysis>   
'||(select xmlelement(name "SeriesMeta",xmlforest(m.display_value as "Value",m.display_unit as "Unit",m.display_label as "DisplayName")) as hi
      from tm_lz.lt_src_display_mapping m where (m.category_cd||m.display_label)=(ul.category_cd||ul.display_label))||
                '</ValueMetadata>') where n.c_fullname in  
				(select leaf_node from wt_trial_nodes where (((category_cd||'+'||replace(data_label,'PCT','%'))||node_name)=(ul.category_cd||ul.display_label) 
				or (category_cd||node_name)=(ul.category_cd||ul.display_label)) and leaf_node=n.c_fullname);
                
                end loop;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Updated I2B2 for metadataXML',rowCt,stepCt,'Done') into rtnCd;

				
	-- inc-change select only updated rows with same patient_num and concept_cd from OBSERVATION_FACT

	--	delete from observation_fact all concept_cds for trial that are clinical data, exclude concept_cds from biomarker data
	
	begin
	delete from i2b2demodata.observation_fact f
	where f.sourcesystem_cd = TrialId
        and (patient_num||concept_cd) in (  select (c.patient_num||i.c_basecode) from wrk_clinical_data a
                                ,patient_dimension c
                                ,wt_trial_nodes t
                                ,i2b2 i
                where a.usubjid = c.sourcesystem_cd
                  and coalesce(a.category_cd,'@') = coalesce(t.category_cd,'@')
                  and coalesce(a.data_label,'**NULL**') = coalesce(t.data_label,'**NULL**')
                  and coalesce(a.visit_name,'**NULL**') = coalesce(t.visit_name,'**NULL**')
--                  and decode(a.data_type,'T',a.data_value,'**NULL**') = coalesce(t.data_value,'**NULL**')
                  and CASE when a.data_type = 'T' THEN a.data_value ELSE '**NULL**' END = coalesce(t.data_value,'**NULL**')
                  and t.leaf_node = i.c_fullname
                  --and c.patient_num not in (select distinct patient_num from observation_fact)
                  and not exists                  -- don't insert if lower level node exists
                                (select 1 from wt_trial_nodes x
                                  --where x.leaf_node like t.leaf_node || '%_'
                                  --Jule 2013. Performance fix by TR. Find if any leaf parent node is current
                                   where (SUBSTR(x.leaf_node, 1, INSTR(x.leaf_node, '\', -2))) = t.leaf_node
                                  
                                  )
                  and a.data_value is not null );

	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Delete clinical data for study from observation_fact',rowCt,stepCt,'Done') into rtnCd;  
	
    --Insert into observation_fact
	
	begin
	insert into i2b2demodata.observation_fact
	(encounter_num,
     patient_num,
     concept_cd,
     start_date,
     modifier_cd,
     valtype_cd,
     tval_char,
     nval_num,
     units_cd,
     sourcesystem_cd,
     import_date,
     valueflag_cd,
     provider_id,
     location_cd,
     instance_num
	)
	select distinct c.patient_num,
		   c.patient_num,
		   i.c_basecode,
		   coalesce(a.date_timestamp, 'infinity'),
		   coalesce(a.modifier_cd, '@'),
		   a.data_type,
		   case when a.data_type = 'T' then a.data_value
				else 'E'  --Stands for Equals for numeric types
				end,
		   case when a.data_type = 'N' then a.data_value::numeric
				else null --Null for text types
				end,
                   a.units_cd,
		   a.study_id, 
		   current_timestamp, 
		   '@',
		   '@',
		   '@',
                   1
	from tm_wz.wrk_clinical_data a
		,i2b2demodata.patient_dimension c
		,tm_wz.wt_trial_nodes t
		,i2b2metadata.i2b2 i
	where a.usubjid = c.sourcesystem_cd
	  and coalesce(a.category_cd,'@') = coalesce(t.category_cd,'@')
	  and coalesce(a.data_label,'**NULL**') = coalesce(t.data_label,'**NULL**')
	  and coalesce(a.visit_name,'**NULL**') = coalesce(t.visit_name,'**NULL**')
	  and case when a.data_type = 'T' then a.data_value else '**NULL**' end = coalesce(t.data_value,'**NULL**')
	  and t.leaf_node = i.c_fullname
	  and not exists		-- don't insert if lower level node exists
		 (select 1 from tm_wz.wt_trial_nodes x
		  where x.leaf_node like t.leaf_node || '%_' escape '`')
	  and a.data_value is not null;
	get diagnostics rowCt := ROW_COUNT; 
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Insert trial into I2B2DEMODATA observation_fact',rowCt,stepCt,'Done') into rtnCd;

	--	update c_visualattributes for all nodes in study, done to pick up node that changed c_columndatatype
	
	begin
	with upd as (select p.c_fullname, count(*) as nbr_children 
				 from i2b2metadata.i2b2 p
					 ,i2b2metadata.i2b2 c
				 where p.c_fullname like topNode || '%' escape '`'
				   and c.c_fullname like p.c_fullname || '%' escape '`'
				 group by p.c_fullname)
	update i2b2metadata.i2b2 b
	set c_visualattributes=case when upd.nbr_children = 1 
								then 'L' || substr(b.c_visualattributes,2,2)
								else 'F' || substr(b.c_visualattributes,2,1) ||
									case when upd.c_fullname = topNode
										then case when highlight_study = 'Y' then 'J' else 'S' end 
									 else substr(b.c_visualattributes,3,1) end
								end
		,c_columndatatype=case when upd.nbr_children > 1 then 'T' else b.c_columndatatype end
	from upd
	where b.c_fullname = upd.c_fullname
	  and b.c_fullname in
		 (select x.c_fullname from i2b2metadata.i2b2 x
		  where x.c_fullname like topNode || '%' escape '`');
  	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Set c_visualattributes in i2b2',rowCt,stepCt,'Done') into rtnCd;

	-- final procs
  
	select tm_cz.i2b2_fill_in_tree(TrialId, topNode, jobID) into rtnCd;
	
	select tm_cz.i2b2_create_concept_counts(topNode, jobID) into rtnCd;
	
	--	delete each node that is hidden after create concept counts
	
	 FOR r_delNodes in delNodes Loop

    --	deletes hidden nodes for a trial one at a time

		select tm_cz.i2b2_delete_1_node(r_delNodes.c_fullname) into rtnCd;
		stepCt := stepCt + 1;
		tText := 'Deleted node: ' || r_delNodes.c_fullname;
		select tm_cz.cz_write_audit(jobId,databaseName,procedureName,tText,rowCt,stepCt,'Done') into rtnCd;

	END LOOP;  	

	select tm_cz.i2b2_create_security_inc_trial(TrialId, secureStudy, jobID) into rtnCd;
	select tm_cz.i2b2_load_security_data(jobID) into rtnCd;
	
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'End i2b2_load_clinical_inc_data',0,stepCt,'Done') into rtnCd;
	
	---Cleanup OVERALL JOB if this proc is being run standalone
	IF newJobFlag = 1
	THEN
		select tm_cz.cz_end_audit (jobID, 'SUCCESS') into rtnCd;
	END IF;

	return 1;
END;

$$;


ALTER FUNCTION tm_cz.i2b2_load_clinical_inc_data(trial_id character varying, top_node character varying, secure_study character varying, highlight_study character varying, currentjobid numeric) OWNER TO tm_cz;

--
