-- Name: i2b2_proteomics_zscore_calc(character varying, character varying, numeric, character varying, numeric, character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_proteomics_zscore_calc(trial_id character varying, run_type character varying DEFAULT 'L'::character varying, currentjobid numeric DEFAULT NULL::numeric, data_type character varying DEFAULT 'R'::character varying, log_base numeric DEFAULT 2, source_cd character varying DEFAULT NULL::character varying) RETURNS numeric
    LANGUAGE plpgsql SECURITY DEFINER
    SET search_path TO 'tm_cz', 'tm_lz', 'tm_wz', 'i2b2demodata', 'i2b2metadata', 'deapp', 'pg_temp'
    AS $$
/*************************************************************************
This Stored Procedure is used in ETL load PROTEOMICS data
Date:12/9/2013
******************************************************************/
Declare
  TrialID character varying(50);
  sourceCD	character varying(50);
  sqlText character varying(2000);
  runType character varying(10);
  dataType character varying(10);
  stgTrial character varying(50);
  idxExists numeric;
  pExists	numeric;
  nbrRecs numeric;
  logBase numeric;
   
  --Audit variables
  newJobFlag numeric(1);
  databaseName VARCHAR(100);
  procedureName VARCHAR(100);
  jobID numeric(18,0);
  stepCt numeric(18,0);
  rtnCd integer;
  rowCt integer;
  
BEGIN

	TrialId := trial_id;
	runType := run_type;
	dataType := data_type;
	logBase := log_base;
	sourceCd := source_cd;
	  
  --Set Audit Parameters
  newJobFlag := 0; -- False (Default)
  jobID := currentJobID;

  databaseName := 'TM_CZ';
  procedureName := 'I2B2_PROTEOMICS_ZSCORE_CALC';

  --Audit JOB Initialization
  --If Job ID does not exist, then this is a single procedure run and we need to create it
  IF(jobID IS NULL or jobID < 1)
  THEN
    newJobFlag := 1; -- True
    select cz_start_audit (procedureName, databaseName) into jobID;
  END IF;
   
  stepCt := 0;
  
	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Starting zscore calc for ' || TrialId || ' RunType: ' || runType || ' dataType: ' || dataType,0,stepCt,'Done') into rtnCd;
  
	if runType != 'L' then
		stepCt := stepCt + 1;
		get diagnostics rowCt := ROW_COUNT;
		select cz_write_audit(jobId,databaseName,procedureName,'Invalid runType passed - procedure exiting',rowCt,stepCt,'Done') into rtnCd;
		select cz_error_handler (jobID, procedureName) into rtnCd;  
		select cz_end_audit (jobID, 'FAIL') into rtnCd;
		return 150;
	end if;
  
--	For Load, make sure that the TrialId passed as parameter is the same as the trial in stg_subject_proteomics_data
--	If not, raise exception

	if runType = 'L' then
		select distinct trial_name into stgTrial
		from WT_SUBJECT_PROTEOMICS_PROBESET;
		
		if stgTrial != TrialId then
			stepCt := stepCt + 1;
			get diagnostics rowCt := ROW_COUNT;
			select cz_write_audit(jobId,databaseName,procedureName,'TrialId not the same as trial in WT_SUBJECT_PROTEOMICS_PROBESET - procedure exiting',rowCt,stepCt,'Done') into rtnCd;
			select cz_error_handler(jobID, procedureName) into rtnCd;
			select cz_end_audit (jobID, 'FAIL') into rtnCd;
			return 161;
		end if;
	end if;
   
--	truncate tmp tables
	begin
		execute ('truncate table tm_wz.WT_SUBJECT_PROTEOMICS_LOGS');
		execute ('truncate table tm_wz.WT_SUBJECT_PROTEOMICS_CALCS');
		execute ('truncate table tm_wz.WT_SUBJECT_PROTEOMICS_MED');
	exception
	when others then
		perform tm_cz.cz_error_handler (jobID, procedureName, SQLSTATE, SQLERRM);
		perform tm_cz.cz_end_audit (jobID, 'FAIL');
		return -16;
	end;
	--drop index if exists tm_wz.WT_SUBJECT_PROTEOMICS_LOGS_I1;		
	--drop index if exists tm_wz.WT_SUBJECT_PROTEOMICS_CALCS_I1;
	
	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Truncate work tables in TM_WZ',0,stepCt,'Done') into rtnCd;
	
	--	if dataType = L, use intensity_value as log_intensity
	--	if dataType = R, always use intensity_value


	if dataType = 'L' then
		begin
		insert into WT_SUBJECT_PROTEOMICS_LOGS 
			(probeset_id
			,intensity_value
			,assay_id
			,log_intensity
			,patient_id
		--	,sample_cd
			,subject_id
			)
			select probeset
				  ,intensity_value ----UAT 154 changes done on 19/03/2014
				  ,assay_id 
				  ,round(intensity_value,4)
				  ,patient_id
			--	  ,sample_cd
				  ,subject_id
			from WT_SUBJECT_PROTEOMICS_PROBESET
			where trial_name = TrialId;
		exception
		when others then
			perform tm_cz.cz_error_handler (jobID, procedureName, SQLSTATE, SQLERRM);
			perform tm_cz.cz_end_audit (jobID, 'FAIL');
			return -16;
		end;
	else	
		begin
                	insert into WT_SUBJECT_PROTEOMICS_LOGS 
			(probeset_id
			,intensity_value
			,assay_id
			,log_intensity
			,patient_id
		--	,sample_cd
			,subject_id
			)
			select probeset
				  ,intensity_value  ----UAT 154 changes done on 19/03/2014
				  ,assay_id 
				  ,round(log(2,intensity_value  + 0.001),4)  ----UAT 154 changes done on 19/03/2014
				  ,patient_id
		--		  ,sample_cd
				  ,subject_id
			from WT_SUBJECT_PROTEOMICS_PROBESET
			where trial_name = TrialId;
		exception
		when others then
			perform tm_cz.cz_error_handler (jobID, procedureName, SQLSTATE, SQLERRM);
			perform tm_cz.cz_end_audit (jobID, 'FAIL');
			return -16;
		end;
	end if;

	stepCt := stepCt + 1;
	get diagnostics rowCt := ROW_COUNT;
	select cz_write_audit(jobId,databaseName,procedureName,'Loaded data for trial in TM_WZ wt_subject_mirna_logs',rowCt,stepCt,'Done') into rtnCd;

	
    
	--execute ('create index WT_SUBJECT_PROTEOMICS_LOGS_I1 on tm_wz.WT_SUBJECT_PROTEOMICS_LOGS (trial_name, probeset_id)');
	stepCt := stepCt + 1;
	--select cz_write_audit(jobId,databaseName,procedureName,'Create index on TM_WZ WT_SUBJECT_PROTEOMICS_LOGS_I1',0,stepCt,'Done') into rtnCd;
		
--	calculate mean_intensity, median_intensity, and stddev_intensity per experiment, probe

	begin
	insert into WT_SUBJECT_PROTEOMICS_CALCS
	(trial_name
	,probeset_id
	,mean_intensity
	,median_intensity
	,stddev_intensity
	)
	select d.trial_name 
		  ,d.probeset_id
		  ,avg(log_intensity)
		  ,median(log_intensity)
		  ,stddev(log_intensity)
	from WT_SUBJECT_PROTEOMICS_LOGS d 
	group by d.trial_name 
			,d.probeset_id;
	exception
	when others then
		perform tm_cz.cz_error_handler (jobID, procedureName, SQLSTATE, SQLERRM);
		perform tm_cz.cz_end_audit (jobID, 'FAIL');
		return -16;
	end;
	stepCt := stepCt + 1;
	get diagnostics rowCt := ROW_COUNT;
	select cz_write_audit(jobId,databaseName,procedureName,'Calculate intensities for trial in TM_WZ WT_SUBJECT_PROTEOMICS_CALCS',rowCt,stepCt,'Done') into rtnCd;

	

	--execute ('create index tm_wz.wt_subject_proteomics_calcs_i1 on tm_wz.WT_SUBJECT_PROTEOMICS_CALCS (trial_name, probeset_id) nologging tablespace "INDX"');
	--stepCt := stepCt + 1;
	--cz_write_audit(jobId,databaseName,procedureName,'Create index on TM_WZ WT_SUBJECT_PROTEOMICS_CALCS',0,stepCt,'Done');
		
-- calculate zscore

	begin
	insert into WT_SUBJECT_PROTEOMICS_MED 
	(probeset_id
	,intensity_value
	,log_intensity
	,assay_id
	,mean_intensity
	,stddev_intensity
	,median_intensity
	,zscore
	,patient_id
--	,sample_cd
	,subject_id
	)
	select d.probeset_id
		  ,d.intensity_value 
		  ,d.log_intensity 
		  ,d.assay_id  
		  ,c.mean_intensity 
		  ,c.stddev_intensity 
		  ,c.median_intensity 
		  ,(CASE WHEN stddev_intensity=0 THEN 0 ELSE (log_intensity - median_intensity ) / stddev_intensity END)
		  ,d.patient_id
	--	  ,d.sample_cd
		  ,d.subject_id
    from WT_SUBJECT_PROTEOMICS_LOGS d 
		,WT_SUBJECT_PROTEOMICS_CALCS c 
    where d.probeset_id = c.probeset_id;
    	exception
	when others then
		perform tm_cz.cz_error_handler (jobID, procedureName, SQLSTATE, SQLERRM);
		perform tm_cz.cz_end_audit (jobID, 'FAIL');
		return -16;
	end;
	stepCt := stepCt + 1;
	get diagnostics rowCt := ROW_COUNT;
	select cz_write_audit(jobId,databaseName,procedureName,'Calculate Z-Score for trial in TM_WZ WT_SUBJECT_PROTEOMICS_MED',rowCt,stepCt,'Done') into rtnCd;

    
	begin
	insert into DE_SUBJECT_PROTEIN_DATA
	(
	 trial_name
	,protein_annotation_id
	,component
	,gene_symbol
	,gene_id
	,assay_id
	,subject_id
	,intensity 
	,zscore
        ,log_intensity
	,patient_id
	)
	select TrialId
                 ,d.id
                 ,m.probeset_id 
                 ,d.uniprot_id
                 ,d.biomarker_id
                 ,m.assay_id
                 ,m.subject_id
	    --  ,decode(dataType,'R',m.intensity_value,'L',power(logBase, m.log_intensity),null)
                ,m.intensity_value as intensity  ---UAT 154 changes done on 19/03/2014
                ,(CASE WHEN m.zscore < -2.5 THEN -2.5 WHEN m.zscore >  2.5 THEN  2.5 ELSE round(m.zscore,5) END)	
                ,round(m.log_intensity,4) as log_intensity
                ,m.patient_id
	from WT_SUBJECT_PROTEOMICS_MED m
       , DEAPP.DE_PROTEIN_ANNOTATION d
        where d.peptide=m.probeset_id;
        exception
	when others then
		perform tm_cz.cz_error_handler (jobID, procedureName, SQLSTATE, SQLERRM);
		perform tm_cz.cz_end_audit (jobID, 'FAIL');
		return -16;
	end;
	stepCt := stepCt + 1;
	get diagnostics rowCt := ROW_COUNT;
	select cz_write_audit(jobId,databaseName,procedureName,'Insert data for trial in DEAPP DE_SUBJECT_PROTEIN_DATA',rowCt,stepCt,'Done') into rtnCd;

   	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Truncate work tables in TM_WZ',0,stepCt,'Done') into rtnCd;
    
    ---Cleanup OVERALL JOB if this proc is being run standalone
  IF newJobFlag = 1
  THEN
    select cz_end_audit (jobID, 'SUCCESS') into rtnCd;
  END IF;
  return 1;	
END;
$$;


ALTER FUNCTION tm_cz.i2b2_proteomics_zscore_calc(trial_id character varying, run_type character varying, currentjobid numeric, data_type character varying, log_base numeric, source_cd character varying) OWNER TO tm_cz;

--
-- Name: i2b2_rbm_zscore_calc_new(character varying, character varying, bigint, character varying, bigint, character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_rbm_zscore_calc_new(trial_id character varying, run_type character varying DEFAULT 'L'::character varying, currentjobid bigint DEFAULT 0, data_type character varying DEFAULT 'R'::character varying, log_base bigint DEFAULT 2, source_cd character varying DEFAULT NULL::character varying) RETURNS void
    LANGUAGE plpgsql
    AS $_$
DECLARE

/*************************************************************************
This Stored Procedure is used in ETL load RBM data
Date:12/04/2013
******************************************************************/

  TrialID varchar(50);
  sourceCD	varchar(50);
  sqlText varchar(2000);
  runType varchar(10);
  dataType varchar(10);
  stgTrial varchar(50);
  idxExists bigint;
  pExists	bigint;
  nbrRecs bigint;
  logBase bigint;
   
  --Audit variables
  newJobFlag integer(1);
  databaseName varchar(100);
  procedureName varchar(100);
  jobID bigint;
  stepCt bigint;
  
  --  exceptions
  invalid_runType exception;
  trial_mismatch exception;
  trial_missing exception;
  

BEGIN

	TrialId := trial_id;
	runType := run_type;
	dataType := data_type;
	logBase := log_base;
	sourceCd := source_cd;
	  
  --Set Audit Parameters
  newJobFlag := 0; -- False (Default)
  jobID := currentJobID;

  PERFORM sys_context('USERENV', 'CURRENT_SCHEMA') INTO databaseName ;
  procedureName := $$PLSQL_UNIT;

  --Audit JOB Initialization
  --If Job ID does not exist, then this is a single procedure run and we need to create it
  IF(coalesce(jobID::text, '') = '' or jobID < 1)
  THEN
    newJobFlag := 1; -- True
    cz_start_audit (procedureName, databaseName, jobID);
  END IF;
   
  stepCt := 0;
  
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Starting zscore calc for ' || TrialId || ' RunType: ' || runType || ' dataType: ' || dataType,0,stepCt,'Done');
  
	if runType != 'L' then
		stepCt := stepCt + 1;
		cz_write_audit(jobId,databaseName,procedureName,'Invalid runType passed - procedure exiting'
,SQL%ROWCOUNT,stepCt,'Done');
		raise invalid_runType;
	end if;
  
--	For Load, make sure that the TrialId passed as parameter is the same as the trial in stg_subject_mrna_data
--	If not, raise exception

	if runType = 'L' then
		select distinct trial_name into stgTrial
		from WT_SUBJECT_RBM_PROBESET;
		
		if stgTrial != TrialId then
			stepCt := stepCt + 1;
			cz_write_audit(jobId,databaseName,procedureName,'TrialId not the same as trial in WT_SUBJECT_RBM_PROBESET - procedure exiting'
,SQL%ROWCOUNT,stepCt,'Done');
			raise trial_mismatch;
		end if;
	end if;

	--remove Reload processing
--	For Reload, make sure that the TrialId passed as parameter has data in de_subject_rbm_data
--	If not, raise exception

	if runType = 'R' then 
		select count(*) into idxExists
		from de_subject_rbm_data
		where trial_name = TrialId;
		
		if idxExists = 0 then
			stepCt := stepCt + 1;
			cz_write_audit(jobId,databaseName,procedureName,'No data for TrialId in de_subject_rbm_data - procedure exiting'
,SQL%ROWCOUNT,stepCt,'Done');
			raise trial_missing;
		end if;
	end if;

   
--	truncate tmp tables

	EXECUTE('truncate table tm_wz.wt_subject_rbm_logs');
	EXECUTE('truncate table tm_wz.wt_subject_rbm_calcs');
	EXECUTE('truncate table tm_wz.wt_subject_rbm_med');

	select count(*) 
	into idxExists
	from all_indexes
	where table_name = 'WT_SUBJECT_RBM_LOGS'
	  and index_name = 'WT_SUBJECT_RBM_LOGS_I1'
	  and owner = 'TM_WZ';
		
	if idxExists = 1 then
		EXECUTE('drop index tm_wz.wt_subject_rbm_logs_i1');		
	end if;
	
	select count(*) 
	into idxExists
	from all_indexes
	where table_name = 'WT_SUBJECT_RBM_CALCS'
	  and index_name = 'WT_SUBJECT_RBM_CALCS_I1'
	  and owner = 'TM_WZ';
		
	if idxExists = 1 then
		EXECUTE('drop index tm_wz.wt_subject_rbm_calcs_i1');
	end if;

	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Truncate work tables in TM_WZ',0,stepCt,'Done');
	
	--	if dataType = L, use intensity_value as log_intensity
	--	if dataType = R, always use intensity_value


	if dataType = 'L' then
		insert into wt_subject_rbm_logs 
			(probeset_id
			,intensity_value
			,assay_id
			,log_intensity
			,patient_id
		--	,sample_cd
		--	,subject_id
			)
			PERFORM probeset
				  ,intensity_value  
				  ,assay_id 
				  ,intensity_value
				  ,patient_id
			--	  ,sample_cd
			--	  ,subject_id
			from wt_subject_rbm_probeset
			where trial_name = TrialId;
           
		--end if;
	else	
                	insert into wt_subject_rbm_logs 
			(probeset_id
			,intensity_value
			,assay_id
			,log_intensity
			,patient_id
		--	,sample_cd
		--	,subject_id
			)
			PERFORM probeset
				  ,intensity_value 
				  ,assay_id 
				  ,log(2,intensity_value)
				  ,patient_id
		--		  ,sample_cd
		--		  ,subject_id
			from wt_subject_rbm_probeset
			where trial_name = TrialId;
--		end if;

	end if;

	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Loaded data for trial in TM_WZ wt_subject_rbm_logs',SQL%ROWCOUNT,stepCt,'Done');

	commit;
    
	EXECUTE('create index tm_wz.wt_subject_rbm_logs_i1 on tm_wz.wt_subject_rbm_logs (trial_name, probeset_id) nologging  tablespace "INDX"');
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Create index on TM_WZ wt_subject_rbm_logs',0,stepCt,'Done');
		
--	calculate mean_intensity, median_intensity, and stddev_intensity per experiment, probe

	insert into wt_subject_rbm_calcs
	(trial_name
	,probeset_id
	,mean_intensity
	,median_intensity
	,stddev_intensity
	)
	PERFORM d.trial_name 
		  ,d.probeset_id
		  ,avg(log_intensity)
		  ,median(log_intensity)
		  ,stddev(log_intensity)
	from wt_subject_rbm_logs d 
	group by d.trial_name 
			,d.probeset_id;
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Calculate intensities for trial in TM_WZ wt_subject_rbm_calcs',SQL%ROWCOUNT,stepCt,'Done');

	commit;

	EXECUTE('create index tm_wz.wt_subject_rbm_calcs_i1 on tm_wz.wt_subject_rbm_calcs (trial_name, probeset_id) nologging tablespace "INDX"');
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Create index on TM_WZ wt_subject_rbm_calcs',0,stepCt,'Done');
		
-- calculate zscore

	insert into wt_subject_rbm_med parallel 
	(probeset_id
	,intensity_value
	,log_intensity
	,assay_id
	,mean_intensity
	,stddev_intensity
	,median_intensity
	,zscore
	,patient_id
--	,sample_cd
--	,subject_id
	)
	PERFORM d.probeset_id
		  ,d.intensity_value 
		  ,d.log_intensity 
		  ,d.assay_id  
		  ,c.mean_intensity 
		  ,c.stddev_intensity 
		  ,c.median_intensity 
		  ,(CASE WHEN stddev_intensity=0 THEN 0 ELSE (log_intensity - median_intensity ) / stddev_intensity END)
		  ,d.patient_id
	--	  ,d.sample_cd
	--	  ,d.subject_id
    from wt_subject_rbm_logs d 
		,wt_subject_rbm_calcs c 
    where d.probeset_id = c.probeset_id;
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Calculate Z-Score for trial in TM_WZ wt_subject_rbm_med',SQL%ROWCOUNT,stepCt,'Done');

    commit;

/*
	select count(*) into n
	select count(*) into nbrRecs
	from wt_subject_rbm_med;
	
	if nbrRecs > 10000000 then
		i2b2_mrna_index_maint('DROP',,jobId);
		stepCt := stepCt + 1;
		cz_write_audit(jobId,databaseName,procedureName,'Drop indexes on DEAPP de_subject_rbm_data',0,stepCt,'Done');
	else
		stepCt := stepCt + 1;
		cz_write_audit(jobId,databaseName,procedureName,'Less than 10M records, index drop bypassed',0,stepCt,'Done');
	end if;
*/
	


	insert into de_subject_rbm_data 
	(trial_name
        --,rbm_annotation_id
	,antigen_name
	,patient_id
        ,gene_symbol
        ,gene_id
        ,assay_id
        ,concept_cd
        ,value
        ,normalized_value
        ,unit    
	,zscore
	)
	PERFORM TrialId
               --,a.id ---rbm_annotation_id
              ,trim(substr(m.probeset_id,1,instr(m.probeset_id,'(')-1)) --m.probeset_id 
              ,m.patient_id
              ,a.gene_symbol  -- gene-symbol 
              ,a.gene_id  --gene_id
              ,m.assay_id
              ,d.concept_code --concept_cd
              ,m.intensity_value
              ,round(case when dataType = 'R' then m.intensity_value
				when dataType = 'L' 
				then case when logBase = -1 then null else power(logBase, m.log_intensity) end
				else null
				end,4) as normalized_value
	    --  ,decode(dataType,'R',m.intensity_value,'L',power(logBase, m.log_intensity),null)
              
              ,trim(substr(m.probeset_id ,instr(m.probeset_id ,'(',-1,1),length(m.probeset_id )))
              ,(CASE WHEN m.zscore < -2.5 THEN -2.5 WHEN m.zscore >  2.5 THEN  2.5 ELSE m.zscore END)
          --	  ,m.sample_id
	--	  ,m.subject_id
	from wt_subject_rbm_med m
        ,wt_subject_rbm_probeset p
        ,DE_RBM_ANNOTATION a
        ,de_subject_sample_mapping d
        where 
        trim(substr(p.probeset,1,instr(p.probeset,'(')-1)) =trim(a.antigen_name) 
        --and 
      and   d.subject_id=p.subject_id
        and p.platform=a.gpl_id
        and m.assay_id=p.assay_id
        and d.platform=p.platform
        and d.patient_id=p.patient_id
        and d.concept_code in (select concept_cd from  i2b2demodata.concept_dimension where concept_cd=d.concept_code)
        and d.trial_name=TrialId
        
        and p.patient_id=m.patient_id
        and p.probeset=m.probeset_id;
        
        insert into DEAPP.DE_RBM_DATA_ANNOTATION_JOIN
select d.id, ann.id from deapp.de_subject_rbm_data d
inner join deapp.de_rbm_annotation ann on ann.antigen_name = d.antigen_name
inner join deapp.de_subject_sample_mapping ssm on ssm.assay_id = d.assay_id and ann.gpl_id = ssm.gpl_id
where not exists( select * from deapp.de_rbm_data_annotation_join j where j.data_id = d.id AND j.annotation_id = ann.id );

        /*
        select distinct TrialId
               ,a.id ---rbm_annotation_id
              ,trim(substr(m.probeset_id,1,instr(m.probeset_id,'(')-1)) --m.probeset_id 
              ,m.patient_id
              ,a.gene_symbol  -- gene-symbol 
              ,a.gene_id  --gene_id
              ,m.assay_id
              ,d.concept_code --concept_cd
              ,m.intensity_value
              ,round(case when dataType = 'R' then m.intensity_value
				when dataType = 'L' 
				then case when logBase = -1 then null else power(logBase, m.log_intensity) end
				else null
				end,4) as normalized_value
	    --  ,decode(dataType,'R',m.intensity_value,'L',power(logBase, m.log_intensity),null)
              
              ,trim(substr(m.probeset_id ,instr(m.probeset_id ,'('),length(m.probeset_id )))
              ,(CASE WHEN m.zscore < -2.5 THEN -2.5 WHEN m.zscore >  2.5 THEN  2.5 ELSE m.zscore END)
          --	  ,m.sample_id
	--	  ,m.subject_id
         from wt_subject_rbm_med m, DE_RBM_ANNOTATION a,de_subject_sample_mapping d
         --(select distinct intensity_value from wt_subject_rbm_probeset p) x
        where a.antigen_name = trim(substr(m.probeset_id,1,instr(m.probeset_id,'(')-1)) and m.patient_id = d.patient_id  
        --and d.assay_id  = m.assay_id
        and d.trial_name = TrialId
        and d.source_cd = source_cd;
        --and d.gpl_id = a.gpl_id;
        --and x.intensity_value = m.intensity_value;
    */
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Insert data for trial in DEAPP de_subject_rbm_data',SQL%ROWCOUNT,stepCt,'Done');

  	commit;

--	add indexes, if indexes were not dropped, procedure will not try and recreate
/*
	i2b2_mrna_index_maint('ADD',,jobId);
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Add indexes on DEAPP de_subject_rbm_data',0,stepCt,'Done');
*/
	
--	cleanup tmp_ files

	--execute immediate('truncate table tm_wz.wt_subject_rbm_logs');
	--execute immediate('truncate table tm_wz.wt_subject_rbm_calcs');
	--execute immediate('truncate table tm_wz.wt_subject_rbm_med');

   	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Truncate work tables in TM_WZ',0,stepCt,'Done');
    
    ---Cleanup OVERALL JOB if this proc is being run standalone
  IF newJobFlag = 1
  THEN 
    cz_end_audit (jobID, 'SUCCESS'); 
  END IF;

  EXCEPTION

  WHEN invalid_runType or trial_mismatch or trial_missing then
    --Handle errors.
    cz_error_handler (jobID, procedureName);
    --End Proc
  
    cz_end_audit (jobID, 'FAIL');
  when OTHERS THEN
    --Handle errors.
    cz_error_handler (jobID, procedureName);


    cz_end_audit (jobID, 'FAIL');
	
END;
 
$_$;


ALTER FUNCTION tm_cz.i2b2_rbm_zscore_calc_new(trial_id character varying, run_type character varying, currentjobid bigint, data_type character varying, log_base bigint, source_cd character varying) OWNER TO tm_cz;

--
-- Name: i2b2_rbm_zscore_calc_new(character varying, character varying, character varying, character varying, bigint, character varying, bigint, character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_rbm_zscore_calc_new(trial_id character varying, partition_name character varying, partition_indx character varying, run_type character varying DEFAULT 'L'::character varying, currentjobid bigint DEFAULT 0, data_type character varying DEFAULT 'R'::character varying, log_base bigint DEFAULT 2, source_cd character varying DEFAULT NULL::character varying) RETURNS void
    LANGUAGE plpgsql
    AS $$
DECLARE

/*************************************************************************
This Stored Procedure is used in ETL load RBM data
Date:12/04/2013
******************************************************************/

  TrialID varchar(50);
  sourceCD	varchar(50);
  sqlText varchar(2000);
  runType varchar(10);
  dataType varchar(10);
  partitionName varchar(200);
  partitionindx varchar(200);
  stgTrial varchar(50);
  idxExists bigint;
  pExists	bigint;
  nbrRecs bigint;
  logBase bigint;
   
  --Audit variables
  newJobFlag integer;
  databaseName varchar(100);
  procedureName varchar(100);
  jobID bigint;
  stepCt bigint;
  rowCt			bigint;
  rtnCd			integer;  
  errorNumber		character varying;
  errorMessage	character varying;

BEGIN

	TrialId := trial_id;
	runType := run_type;
	dataType := data_type;
	logBase := log_base;
	sourceCd := source_cd;
	
	partitionindx := partition_indx;
	partitionName := partition_name;
	  
  --Set Audit Parameters
  newJobFlag := 0; -- False (Default)
  jobID := currentJobID;

	databaseName := 'TM_CZ';
	procedureName := 'I2B2_LOAD_RBM_DATA';

  --Audit JOB Initialization
  --If Job ID does not exist, then this is a single procedure run and we need to create it
  IF(coalesce(jobID::text, '') = '' or jobID < 1)
  THEN
    newJobFlag := 1; -- True
    select cz_start_audit (procedureName, databaseName, jobID) into rtnCd;
  END IF;
   
  stepCt := 0;
  
  	select count(*) into pExists
	from information_schema.tables
	where table_name = partitionindx;

	if pExists = 0 then
	sqlText := 'create table ' || partitionName || ' ( constraint rbm_' || partitionId::text || '_check check ( partition_id = ' || partitionId::text ||
	')) inherits (deapp.de_subject_rbm_data)';
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Create partition ' || partitionName,1,stepCt,'Done') into rtnCd;
	else
	sqlText := 'drop index if exists ' || partitionIndx || '_idx1';
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	sqlText := 'drop index if exists ' || partitionIndx || '_idx2';
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	sqlText := 'drop index if exists ' || partitionIndx || '_idx3';
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	sqlText := 'drop index if exists ' || partitionIndx || '_idx4';
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Drop indexes on ' || partitionName,1,stepCt,'Done') into rtnCd;
	sqlText := 'truncate table ' || partitionName;
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Truncate ' || partitionName,1,stepCt,'Done') into rtnCd;
	end if;
  
	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Starting zscore calc for ' || TrialId || ' RunType: ' || runType || ' dataType: ' || dataType,0,stepCt,'Done') into rtnCd;
  
	if runType != 'L' then
		stepCt := stepCt + 1;
		select cz_write_audit(jobId,databaseName,procedureName,'Invalid runType passed - procedure exiting'
,0,stepCt,'Done') into rtnCd;
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		select cz_end_audit (jobId,'FAIL') into rtnCd;
		return;
	end if;
  
--	For Load, make sure that the TrialId passed as parameter is the same as the trial in stg_subject_mrna_data
--	If not, raise exception

	if runType = 'L' then
		select distinct trial_name into stgTrial
		from tm_wz.WT_SUBJECT_RBM_PROBESET;
		
		if stgTrial != TrialId then
			stepCt := stepCt + 1;
			select cz_write_audit(jobId,databaseName,procedureName,'TrialId not the same as trial in WT_SUBJECT_RBM_PROBESET - procedure exiting'
,0,stepCt,'Done') into rtnCd;
			select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
			select cz_end_audit (jobId,'FAIL') into rtnCd;
			return;
		end if;
	end if;

	--remove Reload processing
--	For Reload, make sure that the TrialId passed as parameter has data in de_subject_rbm_data
--	If not, raise exception

	if runType = 'R' then 
		select count(*) into idxExists
		from deapp.de_subject_rbm_data
		where trial_name = TrialId;
		
		if idxExists = 0 then
			stepCt := stepCt + 1;
			select cz_write_audit(jobId,databaseName,procedureName,'No data for TrialId in de_subject_rbm_data - procedure exiting'
,0,stepCt,'Done') into rtnCd;
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		select cz_end_audit (jobId,'FAIL') into rtnCd;
		return;
		end if;
	end if;

   
--	truncate tmp tables

	EXECUTE('truncate table tm_wz.wt_subject_rbm_logs');
	EXECUTE('truncate table tm_wz.wt_subject_rbm_calcs');
	EXECUTE('truncate table tm_wz.wt_subject_rbm_med');

	EXECUTE('drop index if exists tm_wz.wt_subject_rbm_logs_i1');		
		
	if idxExists = 1 then
		EXECUTE('drop index if exists tm_wz.wt_subject_rbm_calcs_i1');
	end if;

	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Truncate work tables in TM_WZ',0,stepCt,'Done') into rtnCd;
	
	--	if dataType = L, use intensity_value as log_intensity
	--	if dataType = R, always use intensity_value

	begin
		if dataType = 'L' then
			insert into tm_wz.wt_subject_rbm_logs 
				(probeset_id
				,intensity_value
				,assay_id
				,log_intensity
				,patient_id
				)
				select probeset
					  ,intensity_value  
					  ,assay_id 
					  ,intensity_value
					  ,patient_id
				from tm_wz.wt_subject_rbm_probeset
				where trial_name = TrialId;
			   
			--end if;
		else	
			insert into tm_wz.wt_subject_rbm_logs 
				(probeset_id
				,intensity_value
				,assay_id
				,log_intensity
				,patient_id
				)
				select probeset
					  ,intensity_value 
					  ,assay_id 
					  ,log(2,intensity_value)
					  ,patient_id
				from wt_subject_rbm_probeset
				where trial_name = TrialId;

		end if;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return;
	end;
	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Loaded data for trial in TM_WZ wt_subject_rbm_logs',rowCt,stepCt,'Done') into rtnCd;
    
	EXECUTE('create index tm_wz.wt_subject_rbm_logs_i1 on tm_wz.wt_subject_rbm_logs (trial_name, probeset_id) nologging  tablespace "INDX"');
	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Create index on TM_WZ wt_subject_rbm_logs',0,stepCt,'Done') into rtnCd;
		
--	calculate mean_intensity, median_intensity, and stddev_intensity per experiment, probe
	begin
		insert into tm_wz.wt_subject_rbm_calcs
		(trial_name
		,probeset_id
		,mean_intensity
		,median_intensity
		,stddev_intensity
		)
		select d.trial_name 
			  ,d.probeset_id
			  ,avg(log_intensity)
			  ,median(log_intensity)
			  ,stddev(log_intensity)
		from tm_wz.wt_subject_rbm_logs d 
		group by d.trial_name 
				,d.probeset_id;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return;
	end;
	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Calculate intensities for trial in TM_WZ wt_subject_rbm_calcs',SQL%ROWCOUNT,stepCt,'Done') into rtnCd;


	EXECUTE('create index tm_wz.wt_subject_rbm_calcs_i1 on tm_wz.wt_subject_rbm_calcs (trial_name, probeset_id) nologging tablespace "INDX"');
	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Create index on TM_WZ wt_subject_rbm_calcs',0,stepCt,'Done') into rtnCd;
		
-- calculate zscore
	begin
		insert into tm_wz.wt_subject_rbm_med 
		(probeset_id
		,intensity_value
		,log_intensity
		,assay_id
		,mean_intensity
		,stddev_intensity
		,median_intensity
		,zscore
		,patient_id
		)
		select d.probeset_id
			  ,d.intensity_value 
			  ,d.log_intensity 
			  ,d.assay_id  
			  ,c.mean_intensity 
			  ,c.stddev_intensity 
			  ,c.median_intensity 
			  ,(CASE WHEN stddev_intensity=0 THEN 0 ELSE (log_intensity - median_intensity ) / stddev_intensity END)
			  ,d.patient_id
		from tm_wz.wt_subject_rbm_logs d 
			,tm_wz.wt_subject_rbm_calcs c 
		where d.probeset_id = c.probeset_id;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return;
	end;
	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Calculate Z-Score for trial in TM_WZ wt_subject_rbm_med',rowCt,stepCt,'Done') into rtnCd;

	
	-- insert into de_subject_rbm_data when dataType is T (transformed)

	execute ('drop index if exists tm_wz.wt_subject_mrna_logs_i1');
	execute ('drop index if exists tm_wz.wt_subject_mrna_calcs_i1');
	
	sqlText := 'insert into ' || partitionName || 
	'(trial_name, antigen_name, patient_id, gene_symbol, gene_id, assay_id ' ||
        ',concept_cd, value, normalized_value, unit, zscore) ' ||
	'select TrialId ' ||
              ',trim(substr(m.probeset_id,1,instr(m.probeset_id,''('')-1)) --m.probeset_id  ' ||
              ',m.patient_id ' ||
              ',a.gene_symbol  -- gene-symbol  ' ||
              ',a.gene_id  --gene_id ' ||
              ',m.assay_id ' ||
              ',d.concept_code --concept_cd ' ||
              ',m.intensity_value ' ||
              ',round(case when dataType = ''R'' then m.intensity_value ' ||
				'when dataType = ''L''  ' ||
				'then case when logBase = -1 then null else power(logBase, m.log_intensity) end ' ||
				'else null ' ||
				'end,4) as normalized_value ' ||
              ',trim(substr(m.probeset_id ,instr(m.probeset_id ,''('',-1,1),length(m.probeset_id ))) ' ||
              ',(CASE WHEN m.zscore < -2.5 THEN -2.5 WHEN m.zscore >  2.5 THEN  2.5 ELSE m.zscore END) ' ||
	'from wt_subject_rbm_med m ' ||
        ',wt_subject_rbm_probeset p ' ||
        ',DE_RBM_ANNOTATION a ' ||
        ',de_subject_sample_mapping d ' ||
        'where  ' ||
        'trim(substr(p.probeset,1,instr(p.probeset,''('')-1)) =trim(a.antigen_name)  ' ||
      'and   d.subject_id=p.subject_id ' ||
        'and p.platform=a.gpl_id ' ||
        'and m.assay_id=p.assay_id ' ||
        'and d.platform=p.platform ' ||
        'and d.patient_id=p.patient_id ' ||
        'and d.concept_code in (select concept_cd from  i2b2demodata.concept_dimension where concept_cd=d.concept_code) ' ||
        'and d.trial_name=TrialId  ' ||
        'and p.patient_id=m.patient_id ' ||
        'and p.probeset=m.probeset_id ' ;
		
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	get diagnostics rowCt := ROW_COUNT;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Inserted data into ' || partitionName,rowCt,stepCt,'Done') into rtnCd;

	insert into DEAPP.DE_RBM_DATA_ANNOTATION_JOIN
	select d.id, ann.id from deapp.de_subject_rbm_data d
	inner join deapp.de_rbm_annotation ann on ann.antigen_name = d.antigen_name
	inner join deapp.de_subject_sample_mapping ssm on ssm.assay_id = d.assay_id and ann.gpl_id = ssm.gpl_id
	where not exists( select * from deapp.de_rbm_data_annotation_join j where j.data_id = d.id AND j.annotation_id = ann.id );

        
	
	-- create indexes on partition

	sqlText := ' create index ' || partitionIndx || '_idx1 on ' || partitionName || ' using btree (partition_id) tablespace indx';
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	sqlText := ' create index ' || partitionIndx || '_idx2 on ' || partitionName || ' using btree (assay_id) tablespace indx';
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	sqlText := ' create index ' || partitionIndx || '_idx3 on ' || partitionName || ' using btree (probeset_id) tablespace indx';
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	sqlText := ' create index ' || partitionIndx || '_idx4 on ' || partitionName || ' using btree (assay_id, probeset_id) tablespace indx';
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
    
    ---Cleanup OVERALL JOB if this proc is being run standalone
  IF newJobFlag = 1
  THEN 
    select cz_end_audit (jobID, 'SUCCESS') into rtnCd; 
  END IF;

END;
 
$$;


ALTER FUNCTION tm_cz.i2b2_rbm_zscore_calc_new(trial_id character varying, partition_name character varying, partition_indx character varying, run_type character varying, currentjobid bigint, data_type character varying, log_base bigint, source_cd character varying) OWNER TO tm_cz;

--
-- Name: i2b2_rbm_zscore_calc_new(character varying, character varying, character varying, numeric, character varying, bigint, character varying, bigint, character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_rbm_zscore_calc_new(trial_id character varying, partition_name character varying, partition_indx character varying, partitionid numeric, run_type character varying DEFAULT 'L'::character varying, currentjobid bigint DEFAULT 0, data_type character varying DEFAULT 'R'::character varying, log_base bigint DEFAULT 2, source_cd character varying DEFAULT NULL::character varying) RETURNS void
    LANGUAGE plpgsql
    AS $$
DECLARE

/*************************************************************************
This Stored Procedure is used in ETL load RBM data
Date:12/04/2013
******************************************************************/

  TrialID varchar(50);
  sourceCD	varchar(50);
  sqlText varchar(2000);
  runType varchar(10);
  dataType varchar(10);
  partitionName varchar(200);
  partitionindx varchar(200);
  stgTrial varchar(50);
  idxExists bigint;
  pExists	bigint;
  nbrRecs bigint;
  logBase bigint;
   
  --Audit variables
  newJobFlag integer;
  databaseName varchar(100);
  procedureName varchar(100);
  jobID bigint;
  stepCt bigint;
  rowCt			bigint;
  rtnCd			integer;  
  errorNumber		character varying;
  errorMessage	character varying;

BEGIN

	TrialId := trial_id;
	runType := run_type;
	dataType := data_type;
	logBase := log_base;
	sourceCd := source_cd;
	
	partitionindx := partition_indx;
	partitionName := partition_name;
	  
  --Set Audit Parameters
  newJobFlag := 0; -- False (Default)
  jobID := currentJobID;

	databaseName := 'TM_CZ';
	procedureName := 'i2b2_rbm_zscore_calc_new';

  --Audit JOB Initialization
  --If Job ID does not exist, then this is a single procedure run and we need to create it
  IF(coalesce(jobID::text, '') = '' or jobID < 1)
  THEN
    newJobFlag := 1; -- True
    select cz_start_audit (procedureName, databaseName, jobID) into rtnCd;
  END IF;
   
  stepCt := 0;
  
  	select count(*) into pExists
	from information_schema.tables
	where table_name = partitionindx;

	if pExists = 0 then
	sqlText := 'create table ' || partitionName || ' ( constraint rbm_' || partitionId::text || '_check check ( partition_id = ' || partitionId::text ||
	')) inherits (deapp.de_subject_rbm_data)';
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Create partition ' || partitionName,1,stepCt,'Done') into rtnCd;
	else
	sqlText := 'drop index if exists ' || partitionIndx || '_idx1';
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	sqlText := 'drop index if exists ' || partitionIndx || '_idx2';
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	sqlText := 'drop index if exists ' || partitionIndx || '_idx3';
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	sqlText := 'drop index if exists ' || partitionIndx || '_idx4';
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Drop indexes on ' || partitionName,1,stepCt,'Done') into rtnCd;
	sqlText := 'truncate table ' || partitionName;
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Truncate ' || partitionName,1,stepCt,'Done') into rtnCd;
	end if;
  
	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Starting zscore calc for ' || TrialId || ' RunType: ' || runType || ' dataType: ' || dataType,0,stepCt,'Done') into rtnCd;
  
	if runType != 'L' then
		stepCt := stepCt + 1;
		select cz_write_audit(jobId,databaseName,procedureName,'Invalid runType passed - procedure exiting'
,0,stepCt,'Done') into rtnCd;
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		select cz_end_audit (jobId,'FAIL') into rtnCd;
		return;
	end if;
  
--	For Load, make sure that the TrialId passed as parameter is the same as the trial in stg_subject_mrna_data
--	If not, raise exception

	if runType = 'L' then
		select distinct trial_name into stgTrial
		from tm_wz.WT_SUBJECT_RBM_PROBESET;
		
		if stgTrial != TrialId then
			stepCt := stepCt + 1;
			select cz_write_audit(jobId,databaseName,procedureName,'TrialId not the same as trial in WT_SUBJECT_RBM_PROBESET - procedure exiting'
,0,stepCt,'Done') into rtnCd;
			select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
			select cz_end_audit (jobId,'FAIL') into rtnCd;
			return;
		end if;
	end if;

	--remove Reload processing
--	For Reload, make sure that the TrialId passed as parameter has data in de_subject_rbm_data
--	If not, raise exception

	if runType = 'R' then 
		select count(*) into idxExists
		from deapp.de_subject_rbm_data
		where trial_name = TrialId;
		
		if idxExists = 0 then
			stepCt := stepCt + 1;
			select cz_write_audit(jobId,databaseName,procedureName,'No data for TrialId in de_subject_rbm_data - procedure exiting'
,0,stepCt,'Done') into rtnCd;
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		select cz_end_audit (jobId,'FAIL') into rtnCd;
		return;
		end if;
	end if;

   
--	truncate tmp tables

	EXECUTE('truncate table tm_wz.wt_subject_rbm_logs');
	EXECUTE('truncate table tm_wz.wt_subject_rbm_calcs');
	EXECUTE('truncate table tm_wz.wt_subject_rbm_med');

	EXECUTE('drop index if exists tm_wz.wt_subject_rbm_logs_i1');		
		
	EXECUTE('drop index if exists tm_wz.wt_subject_rbm_calcs_i1');
	
	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Truncate work tables in TM_WZ',0,stepCt,'Done') into rtnCd;
	
	--	if dataType = L, use intensity_value as log_intensity
	--	if dataType = R, always use intensity_value

	begin
		if dataType = 'L' then
			insert into tm_wz.wt_subject_rbm_logs 
				(probeset_id
				,intensity_value
				,assay_id
				,log_intensity
				,patient_id
				)
				select probeset
					  ,intensity_value  
					  ,assay_id 
					  ,intensity_value
					  ,patient_id
				from tm_wz.wt_subject_rbm_probeset
				where trial_name = TrialId;
			   
			--end if;
		else	
			insert into tm_wz.wt_subject_rbm_logs 
				(probeset_id
				,intensity_value
				,assay_id
				,log_intensity
				,patient_id
				)
				select probeset
					  ,intensity_value 
					  ,assay_id 
					  ,CASE WHEN intensity_value <= 0 THEN log(2,(intensity_value + 0.001)) ELSE log(2,intensity_value) END
					  ,patient_id
				from tm_wz.wt_subject_rbm_probeset
				where trial_name = TrialId;
		end if;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return;
	end;
	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Loaded data for trial in TM_WZ wt_subject_rbm_logs',rowCt,stepCt,'Done') into rtnCd;
    
	EXECUTE('create index wt_subject_rbm_logs_i1 on tm_wz.wt_subject_rbm_logs (trial_name, probeset_id) tablespace "indx"');
	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Create index on TM_WZ wt_subject_rbm_logs',0,stepCt,'Done') into rtnCd;
		
--	calculate mean_intensity, median_intensity, and stddev_intensity per experiment, probe
	begin
		insert into tm_wz.wt_subject_rbm_calcs
		(trial_name
		,probeset_id
		,mean_intensity
		,median_intensity
		,stddev_intensity
		)
		select d.trial_name 
			  ,d.probeset_id
			  ,avg(log_intensity)
			  ,median(log_intensity)
			  ,stddev(log_intensity)
		from tm_wz.wt_subject_rbm_logs d 
		group by d.trial_name 
				,d.probeset_id;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return;
	end;
	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Calculate intensities for trial in TM_WZ wt_subject_rbm_calcs',rowCt,stepCt,'Done') into rtnCd;


	EXECUTE('create index wt_subject_rbm_calcs_i1 on tm_wz.wt_subject_rbm_calcs (trial_name, probeset_id) tablespace "indx"');
	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Create index on TM_WZ wt_subject_rbm_calcs',0,stepCt,'Done') into rtnCd;
		
-- calculate zscore
	begin
		insert into tm_wz.wt_subject_rbm_med 
		(probeset_id
		,intensity_value
		,log_intensity
		,assay_id
		,mean_intensity
		,stddev_intensity
		,median_intensity
		,zscore
		,patient_id
		)
		select d.probeset_id
			  ,d.intensity_value 
			  ,d.log_intensity 
			  ,d.assay_id  
			  ,c.mean_intensity 
			  ,c.stddev_intensity 
			  ,c.median_intensity 
			  ,(CASE WHEN stddev_intensity=0 THEN 0 ELSE (log_intensity - median_intensity ) / stddev_intensity END)
			  ,d.patient_id
		from tm_wz.wt_subject_rbm_logs d 
			,tm_wz.wt_subject_rbm_calcs c 
		where d.probeset_id = c.probeset_id;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return;
	end;
	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Calculate Z-Score for trial in TM_WZ wt_subject_rbm_med',rowCt,stepCt,'Done') into rtnCd;

	
	-- insert into de_subject_rbm_data when dataType is T (transformed)

	
	sqlText := 'insert into ' || partitionName || 
	'(partition_id, trial_name, antigen_name, patient_id, gene_symbol, gene_id, assay_id ' ||
        ',concept_cd, value, normalized_value, unit, zscore, id) ' ||
	'select ' || partitioniD::text || ', ''' || TrialId || '''' ||
              ',trim(substr(m.probeset_id,1,instr(m.probeset_id,''('')-1)) ' ||
              ',m.patient_id ' ||
              ',a.gene_symbol  ' ||
              ',a.gene_id::integer  ' ||
              ',m.assay_id ' ||
              ',d.concept_code ' ||
              ',m.intensity_value ' ||
              ',round(case when ''' || dataType || ''' = ''R'' then m.intensity_value::numeric ' ||
				'when ''' || dataType || ''' = ''L''  ' ||
				'then case when ''' || logBase || ''' = -1 then null else power( ''' || logBase || ''' , m.log_intensity)::numeric end ' ||
				'else null ' ||
				'end,4) as normalized_value ' ||
              ',trim(substr(m.probeset_id ,instr(m.probeset_id ,''('',-1,1),length(m.probeset_id ))) ' ||
              ',(CASE WHEN m.zscore < -2.5 THEN -2.5 WHEN m.zscore >  2.5 THEN  2.5 ELSE m.zscore END) ' ||
			  ',nextval(''deapp.RBM_ANNOTATION_ID'') '||
	'from tm_wz.wt_subject_rbm_med m ' ||
        ',tm_wz.wt_subject_rbm_probeset p ' ||
        ',deapp.DE_RBM_ANNOTATION a ' ||
        ',deapp.de_subject_sample_mapping d ' ||
        'where  ' ||
        'trim(substr(p.probeset,1,instr(p.probeset,''('')-1)) =trim(a.antigen_name)  ' ||
      'and   d.subject_id=p.subject_id ' ||
        'and p.platform=a.gpl_id ' ||
        'and m.assay_id=p.assay_id ' ||
        'and d.gpl_id=p.platform ' ||
        'and d.patient_id=p.patient_id ' ||
        'and d.concept_code in (select concept_cd from  i2b2demodata.concept_dimension where concept_cd=d.concept_code) ' ||
        'and d.trial_name=''' || TrialId || '''' ||
        'and p.patient_id=m.patient_id ' ||
        'and p.probeset=m.probeset_id ' ;
		
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	get diagnostics rowCt := ROW_COUNT;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Inserted data into ' || partitionName,rowCt,stepCt,'Done') into rtnCd;

	insert into DEAPP.DE_RBM_DATA_ANNOTATION_JOIN
	select d.id, ann.id from deapp.de_subject_rbm_data d
	inner join deapp.de_rbm_annotation ann on ann.antigen_name = d.antigen_name
	inner join deapp.de_subject_sample_mapping ssm on ssm.assay_id = d.assay_id and ann.gpl_id = ssm.gpl_id
	where not exists( select * from deapp.de_rbm_data_annotation_join j where j.data_id = d.id AND j.annotation_id = ann.id );
    
    ---Cleanup OVERALL JOB if this proc is being run standalone
  IF newJobFlag = 1
  THEN 
    select cz_end_audit (jobID, 'SUCCESS') into rtnCd; 
  END IF;

END;
 
$$;


ALTER FUNCTION tm_cz.i2b2_rbm_zscore_calc_new(trial_id character varying, partition_name character varying, partition_indx character varying, partitionid numeric, run_type character varying, currentjobid bigint, data_type character varying, log_base bigint, source_cd character varying) OWNER TO tm_cz;

--
-- Name: i2b2_rename_node(character varying, character varying, character varying, numeric); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_rename_node(trial_id character varying, old_node character varying, new_node character varying, currentjobid numeric DEFAULT '-1'::integer) RETURNS void
    LANGUAGE plpgsql
    AS $_$
DECLARE

/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
   
  --Audit variables
  newJobFlag integer(1);
  databaseName varchar(100);
  procedureName varchar(100);
  jobID bigint;
  stepCt bigint;
  

BEGIN
	
  --Set Audit Parameters
  newJobFlag := 0; -- False (Default)
  jobID := currentJobID;

  PERFORM sys_context('USERENV', 'CURRENT_SCHEMA') INTO databaseName ;
  procedureName := $$PLSQL_UNIT;

  --Audit JOB Initialization
  --If Job ID does not exist, then this is a single procedure run and we need to create it
  IF(coalesce(jobID::text, '') = '' or jobID < 1)
  THEN
    newJobFlag := 1; -- True
    cz_start_audit (procedureName, databaseName, jobID);
  END IF;
    	
  stepCt := 0;

  
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Start i2b2_rename_node',0,stepCt,'Done'); 
	
  if old_node != ''  and old_node != '%' and new_node != ''  and new_node != '%'
  then

	--	Update concept_counts paths

	update concept_counts cc
      set CONCEPT_PATH = replace(cc.concept_path, '\' || old_node || '\', '\' || new_node || '\'),
	      parent_concept_path = replace(cc.parent_concept_path, '\' || old_node || '\', '\' || new_node || '\')
      where cc.concept_path in
		   (select cd.concept_path from concept_dimension cd
		    where cd.sourcesystem_cd = trial_id
              and cd.concept_path like '%' || old_node || '%');
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Update concept_counts with new path',SQL%ROWCOUNT,stepCt,'Done'); 

    COMMIT;
	
    --Update path in i2b2_tags
    update i2b2_tags t
      set path = replace(t.path, '\' || old_node || '\', '\' || new_node || '\')
      where t.path in
		   (select cd.concept_path from concept_dimension cd
		    where cd.sourcesystem_cd = trial_id
              and cd.concept_path like '%\' || old_node || '\%');
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Update i2b2_tags with new path',SQL%ROWCOUNT,stepCt,'Done'); 

    COMMIT;
	
    --Update specific name
    --update concept_dimension
    --  set name_char = new_node
    --  where name_char = old_node
    --    and sourcesystem_cd = trial_id;

    --Update all paths
    update concept_dimension
      set CONCEPT_PATH = replace(concept_path, '\' || old_node || '\', '\' || new_node || '\')
	     ,name_char=(CASE WHEN name_char=old_node THEN new_node ELSE name_char END)
      where
		sourcesystem_cd = trial_id
        and concept_path like '%\' || old_node || '\%';
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Update concept_dimension with new path',SQL%ROWCOUNT,stepCt,'Done'); 

    COMMIT;



    --I2B2
    --Update specific name
    --update i2b2
    --  set c_name = new_node
    --  where c_name = old_node
    --    and c_fullname like '%' || trial_id || '%';

    --Update all paths, added updates to c_dimcode and c_tooltip instead of separate pass
    update i2b2
      set c_fullname = replace(c_fullname, '\' || old_node || '\', '\' || new_node || '\')
	  	 ,c_dimcode = replace(c_dimcode, '\' || old_node || '\', '\' || new_node || '\')
		 ,c_tooltip = replace(c_tooltip, '\' || old_node || '\', '\' || new_node || '\')
		 ,c_name = (CASE WHEN c_name=old_node THEN new_node ELSE c_name END)
      where sourcesystem_cd = trial_id
        and c_fullname like '%\' || old_node || '\%';
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Update i2b2 with new path',SQL%ROWCOUNT,stepCt,'Done'); 

    COMMIT;

	--Update i2b2_secure to match i2b2
    --update i2b2_secure
    --  set c_fullname = replace(c_fullname, old_node, new_node)
	--  	 ,c_dimcode = replace(c_dimcode, old_node, new_node)
	--	 ,c_tooltip = replace(c_tooltip, old_node, new_node)
    --  where
    --    c_fullname like '%' || trial_id || '%';
    --COMMIT;
	
	i2b2_load_security_data(jobID);


  END IF;
END;
 
$_$;


ALTER FUNCTION tm_cz.i2b2_rename_node(trial_id character varying, old_node character varying, new_node character varying, currentjobid numeric) OWNER TO tm_cz;

--
-- Name: i2b2_rna_seq_annotation(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_rna_seq_annotation() RETURNS bigint
    LANGUAGE plpgsql
    AS $$
DECLARE

	gpl_rtn bigint;
	newJobFlag numeric(1);
	databaseName character varying(100);
	procedureName character varying(100);
	jobID bigint;
	errorNumber		character varying;
	errorMessage	character varying;
	rtnCd			integer;
	rowCt			numeric(18,0);
	stepCt bigint;
BEGIN

	stepCt := 0;

	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := -1;
	
	databaseName := 'TM_CZ';
	procedureName := 'I2B2_RBA_SEQ_ANNOTATION';

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it
	IF(coalesce(jobID::text, '') = '' or jobID < 1)
	THEN
		newJobFlag := 1; -- True
		select cz_start_audit (procedureName, databaseName, jobID) into jobId;
	END IF;

	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Starting i2b2_rna_seq_annotation',0,stepCt,'Done') into rtnCd;
	
 -- insert into "DEAPP"."DE_RNASEQ_ANNOTATION"
 
 select count(platform) into gpl_rtn from deapp.de_gpl_info where marker_type='RNASEQ' and (platform IS NOT NULL AND platform::text <> '');
 if gpl_rtn=0 then
		select cz_write_audit(jobId,databasename,procedurename,'Platform data missing from DEAPP.DE_GPL_INFO',1,stepCt,'ERROR') into rtnCd;
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		select cz_end_audit (jobId,'FAIL') into rtnCd;
		return 161;
 end if;
 
 begin
 insert into deapp.DE_RNASEQ_ANNOTATION 
 (
   GPL_ID
  ,TRANSCRIPT_ID 
  ,GENE_SYMBOL
  ,GENE_ID
  ,ORGANISM
  ,PROBESET_ID
    )
    select g.platform
          ,a.transcript_id
          ,a.gene_symbol
          ,b.bio_marker_id
          ,a.organism
          ,pd.probeset_id
          from tm_lz.LT_RNASEQ_ANNOTATION a
               ,(select platform from deapp.de_gpl_info where marker_type='RNASEQ') as g
               ,biomart.bio_marker b
               ,probeset_deapp pd
                where b.bio_marker_name=a.gene_symbol
                and a.transcript_id =pd.probeset;
	get diagnostics rowCt := ROW_COUNT;	
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	
	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Insert new probesets into antigen_deapp',rowCt,stepCt,'Done') into rtnCd;
	
   return 0 ;
              
 
END;
 
$$;


ALTER FUNCTION tm_cz.i2b2_rna_seq_annotation() OWNER TO tm_cz;

--
-- Name: i2b2_rna_seq_annotation(bigint); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_rna_seq_annotation(currentjobid bigint DEFAULT NULL::bigint) RETURNS numeric
    LANGUAGE plpgsql
    AS $$
DECLARE

	gpl_rtn bigint;
	newJobFlag numeric(1);
	databaseName character varying(100);
	procedureName character varying(100);
	jobID bigint;
	errorNumber		character varying;
	errorMessage	character varying;
	rtnCd			integer;
	rowCt			numeric(18,0);
	stepCt bigint;
BEGIN

	stepCt := 0;

	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;
	
	databaseName := 'TM_CZ';
	procedureName := 'I2B2_RNA_SEQ_ANNOTATION';

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it
	IF(coalesce(jobID::text, '') = '' or jobID < 1)
	THEN
		newJobFlag := 1; -- True
		select cz_start_audit (procedureName, databaseName, jobID) into jobId;
	END IF;

	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Starting i2b2_rna_seq_annotation',0,stepCt,'Done') into rtnCd;
	
 -- insert into "DEAPP"."DE_RNASEQ_ANNOTATION"
 
 select count(platform) into gpl_rtn from deapp.de_gpl_info where marker_type='RNASEQ' and (platform IS NOT NULL AND platform::text <> '');
 if gpl_rtn=0 then
		select cz_write_audit(jobId,databasename,procedurename,'Platform data missing from DEAPP.DE_GPL_INFO',1,stepCt,'ERROR') into rtnCd;
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		select cz_end_audit (jobId,'FAIL') into rtnCd;
		return 161;
 end if;
 
 begin
 insert into deapp.DE_RNASEQ_ANNOTATION
 (
 TRANSCRIPT_ID
   ,GPL_ID
    ,GENE_SYMBOL
  ,GENE_ID
  ,ORGANISM
    )
    select distinct (a.transcript_id)
      ,null
          ,a.gene_symbol
          ,null
          ,a.organism
          from tm_lz.LT_RNASEQ_ANNOTATION a
                where a.transcript_id not in (select distinct transcript_id from deapp.DE_RNASEQ_ANNOTATION);
	get diagnostics rowCt := ROW_COUNT;	
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	
	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Insert data in DE_RNASEQ_ANNOTATION',0,stepCt,'Done') into rtnCd;
	
	begin
    update deapp.DE_RNASEQ_ANNOTATION a set GENE_ID=(select distinct primary_external_id from biomart.bio_marker b where
	b.bio_marker_name=a.gene_symbol limit 1)
    where a.GENE_ID is null;
	get diagnostics rowCt := ROW_COUNT;	
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return -16;
	end;
	
	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'End i2b2_rna_seq_annotation',0,stepCt,'Done') into rtnCd;
	
       ---Cleanup OVERALL JOB if this proc is being run standalone
  IF newJobFlag = 1
  THEN
    select cz_end_audit (jobID, 'SUCCESS') into rtnCd;
  END IF;

  return 0;
 
END;
 
$$;


ALTER FUNCTION tm_cz.i2b2_rna_seq_annotation(currentjobid bigint) OWNER TO tm_cz;

--
-- Name: i2b2_rna_seq_zscore_calc(character varying, character varying, character varying, numeric, character varying, numeric, character varying, bigint, character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_rna_seq_zscore_calc(trial_id character varying, partition_name character varying, partition_indx character varying, partitionid numeric, run_type character varying DEFAULT 'L'::character varying, currentjobid numeric DEFAULT '-1'::integer, data_type character varying DEFAULT 'R'::character varying, log_base bigint DEFAULT 2, source_cd character varying DEFAULT NULL::character varying) RETURNS void
    LANGUAGE plpgsql
    AS $$
DECLARE

/*************************************************************************
* This stored procedure is used to calculate z-scores for RNA sequencing data load
* Date:10/23/2013
******************************************************************/

  TrialID varchar(50);
  sourceCD	varchar(50);
  sqlText varchar(2000);
  runType varchar(10);
  dataType varchar(10);
  stgTrial varchar(50);
  idxExists bigint;
  pExists	bigint;
  nbrRecs bigint;
  logBase bigint;
  partitionName varchar(200);
  partitionindx varchar(200);
   
  --Audit variables
  newJobFlag integer;
  databaseName varchar(100);
  procedureName varchar(100);
  jobID bigint;
  stepCt bigint;
  rowCt			bigint;
  rtnCd			integer;  
  errorNumber		character varying;
  errorMessage	character varying;


BEGIN

	TrialId := trial_id;
	runType := run_type;
	dataType := data_type;
	logBase := log_base;
	sourceCd := source_cd;

	partitionindx := partition_indx;
	partitionName := partition_name;	
  --Set Audit Parameters
  newJobFlag := 0; -- False (Default)
  jobID := currentJobID;

  	databaseName := 'TM_CZ';
	procedureName := 'i2b2_rna_seq_zscore_calc';

  --Audit JOB Initialization
  --If Job ID does not exist, then this is a single procedure run and we need to create it
  IF(coalesce(jobID::text, '') = '' or jobID < 1)
  THEN
    newJobFlag := 1; -- True
    select cz_start_audit (procedureName, databaseName, jobID) into rtnCd;
  END IF;
    	
  stepCt := 0;
  
    	select count(*) into pExists
	from information_schema.tables
	where table_name = partitionindx;

	if pExists = 0 then
	sqlText := 'create table ' || partitionName || ' ( constraint rnaseq_' || partitionId::text || '_check check ( partition_id = ' || partitionId::text ||
	')) inherits (deapp.de_subject_rna_data)';
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Create partition ' || partitionName,1,stepCt,'Done') into rtnCd;
	else
	sqlText := 'drop index if exists ' || partitionIndx || '_idx1';
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	sqlText := 'drop index if exists ' || partitionIndx || '_idx2';
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	sqlText := 'drop index if exists ' || partitionIndx || '_idx3';
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	sqlText := 'drop index if exists ' || partitionIndx || '_idx4';
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Drop indexes on ' || partitionName,1,stepCt,'Done') into rtnCd;
	sqlText := 'truncate table ' || partitionName;
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Truncate ' || partitionName,1,stepCt,'Done') into rtnCd;
	end if;
  
	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Starting zscore calc for ' || TrialId || ' RunType: ' || runType || ' dataType: ' || dataType,0,stepCt,'Done') into rtnCd;
  
	if runType != 'L' then
		stepCt := stepCt + 1;
		select cz_write_audit(jobId,databaseName,procedureName,'Invalid runType passed - procedure exiting'
,0,stepCt,'Done') into rtnCd;
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		select cz_end_audit (jobId,'FAIL') into rtnCd;
		return;
	end if;

	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Starting zscore calc for ' || TrialId || ' RunType: ' || runType || ' dataType: ' || dataType,0,stepCt,'Done') into rtnCd;
  
	if runType != 'L' then
		stepCt := stepCt + 1;
		select cz_write_audit(jobId,databaseName,procedureName,'Invalid runType passed - procedure exiting'
,0,stepCt,'Done') into rtnCd;
		select cz_error_handler (jobID, procedureName) into rtnCd;
		select cz_end_audit (jobID, 'FAIL') into rtnCd;
		return;
	end if;
  
	if runType = 'L' then
		select distinct trial_name into stgTrial
		from tm_wz.wt_subject_rna_probeset;
		
		if stgTrial != TrialId then
			stepCt := stepCt + 1;
			select cz_write_audit(jobId,databaseName,procedureName,'TrialId not the same as trial in wt_subject_rna_probeset - procedure exiting'
,0,stepCt,'Done') into rtnCd;
			select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
			select cz_end_audit (jobId,'FAIL') into rtnCd;
			return;
		end if;
	end if;
   
--	truncate tmp tables

	EXECUTE('truncate table tm_wz.wt_subject_rna_logs');
	EXECUTE('truncate table tm_wz.wt_subject_rna_calcs');
	EXECUTE('truncate table tm_wz.wt_subject_rna_med');
	
	EXECUTE('drop index if exists tm_wz.WT_SUBJECT_RNA_LOGS_I1');		
	EXECUTE('drop index if exists tm_wz.WT_SUBJECT_RNA_CALCS_I1');

	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Truncate work tables in TM_WZ',0,stepCt,'Done') into rtnCd;
	
	--	if dataType = L, use intensity_value as log_intensity
	--	if dataType = R, always use intensity_value
	begin
	if dataType = 'L' then
			insert into tm_wz.wt_subject_rna_logs 
			(probeset_id
			,intensity_value
			,assay_id
			,log_intensity
			,patient_id
		--	,sample_cd
		--	,subject_id
			)
			select probeset_id
				  ,intensity_value  
				  ,assay_id 
				  ,intensity_value
				  ,patient_id
			from tm_wz.wt_subject_rna_probeset
			where trial_name = TrialId;
			
	else
			insert into tm_wz.wt_subject_rna_logs 
			(probeset_id
			,intensity_value
			,assay_id
			,log_intensity
			,patient_id
			)
			select probeset_id
				  ,intensity_value 
				  ,assay_id 
				  ,CASE WHEN intensity_value <= 0 THEN log(2,(intensity_value + 0.001)) ELSE log(2,intensity_value) END
				  ,patient_id
			from tm_wz.wt_subject_rna_probeset
			where trial_name = TrialId;
	end if;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return;
	end;

	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Loaded data for trial in TM_WZ wt_subject_rna_logs',rowCt,stepCt,'Done') into rtnCd;

	begin
	EXECUTE('create index WT_SUBJECT_RNA_LOGS_I1 on tm_wz.wt_subject_rna_logs (trial_name, probeset_id) tablespace "indx"');
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return;
	end;
	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Create index on TM_WZ WT_SUBJECT_RNA_LOGS_I1',0,stepCt,'Done') into rtnCd;
		
--	calculate mean_intensity, median_intensity, and stddev_intensity per experiment, probe
	begin
	insert into tm_wz.wt_subject_rna_calcs
	(trial_name
	,probeset_id
	,mean_intensity
	,median_intensity
	,stddev_intensity
	)
	select d.trial_name 
		  ,d.probeset_id
		  ,avg(log_intensity)
		  ,median(log_intensity)
		  ,stddev(log_intensity)
	from tm_wz.wt_subject_rna_logs d 
	group by d.trial_name 
			,d.probeset_id;
			get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return;
	end;
	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Calculate intensities for trial in TM_WZ wt_subject_rna_calcs',rowCt,stepCt,'Done') into rtnCd;
	
	begin
	EXECUTE('create index WT_SUBJECT_RNA_CALCS_I1 on tm_wz.wt_subject_rna_calcs (trial_name, probeset_id) tablespace "indx"');
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return;
	end;
	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Create index on TM_WZ WT_SUBJECT_RNA_CALCS_I1',0,stepCt,'Done') into rtnCd;
		
-- calculate zscore
	begin
	insert into tm_wz.wt_subject_rna_med 
	(probeset_id
	,intensity_value
	,log_intensity
	,assay_id
	,mean_intensity
	,stddev_intensity
	,median_intensity
	,zscore
	,patient_id
	)
	select d.probeset_id
		  ,d.intensity_value 
		  ,d.log_intensity 
		  ,d.assay_id  
		  ,c.mean_intensity 
		  ,c.stddev_intensity 
		  ,c.median_intensity 
		  ,CASE WHEN stddev_intensity=0 THEN 0 ELSE (log_intensity - median_intensity ) / stddev_intensity END
		  ,d.patient_id
    from tm_wz.wt_subject_rna_logs d 
		,tm_wz.wt_subject_rna_calcs c 
    where d.probeset_id = c.probeset_id;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return;
	end;
	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Calculate Z-Score for trial in TM_WZ wt_subject_rna_med',rowCt,stepCt,'Done') into rtnCd;
  
	begin
	sqlText := 'insert into ' || partitionName || 
	'(partition_id, trial_source,trial_name,assay_id,probeset_id,raw_intensity ' ||
	',log_intensity,zscore,patient_id) '||
	'select ' || partitioniD::text || ', ''' || TrialId || '''' ||
		  ',''' || TrialId || ''',m.assay_id,m.probeset_id ' ||
		  ',round((case when ''' || dataType || '''= ''R'' then m.intensity_value::numeric '||
				'when ''' || dataType || '''= ''L'' ' ||
				'then case when ' || logBase || '= -1 then null else power(' || logBase || ', m.log_intensity)::numeric end ' ||
				'else null end),4) as raw_intensity ' ||
		  ',round(m.log_intensity::numeric,4) ' ||
	      ',round(CASE WHEN m.zscore < -2.5 THEN -2.5 WHEN m.zscore >  2.5 THEN  2.5 ELSE round(m.zscore::numeric,5) END,5) ' ||
		  ',m.patient_id ' ||
	'from tm_wz.wt_subject_rna_med m';
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	get diagnostics rowCt := ROW_COUNT;
	exception
	when others then
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		--Handle errors.
		select tm_cz.cz_error_handler (jobID, procedureName, errorNumber, errorMessage) into rtnCd;
		--End Proc
		select tm_cz.cz_end_audit (jobID, 'FAIL') into rtnCd;
		return;
	end;
	stepCt := stepCt + 1;
	select tm_cz.cz_write_audit(jobId,databaseName,procedureName,'Inserted data into ' || partitionName,rowCt,stepCt,'Done') into rtnCd;

	EXECUTE('truncate table tm_wz.wt_subject_rna_logs');
	EXECUTE('truncate table tm_wz.wt_subject_rna_calcs');
	EXECUTE('truncate table tm_wz.wt_subject_rna_med');

   	stepCt := stepCt + 1;
	select cz_write_audit(jobId,databaseName,procedureName,'Truncate work tables in TM_WZ',0,stepCt,'Done') into rtnCd;
	
		-- create indexes on partition

	sqlText := ' create index ' || partitionIndx || '_idx1 on ' || partitionName || ' using btree (partition_id) tablespace indx';
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	sqlText := ' create index ' || partitionIndx || '_idx2 on ' || partitionName || ' using btree (assay_id) tablespace indx';
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	sqlText := ' create index ' || partitionIndx || '_idx3 on ' || partitionName || ' using btree (probeset_id) tablespace indx';
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
	sqlText := ' create index ' || partitionIndx || '_idx4 on ' || partitionName || ' using btree (assay_id, probeset_id) tablespace indx';
	raise notice 'sqlText= %', sqlText;
	execute sqlText;
    
    ---Cleanup OVERALL JOB if this proc is being run standalone
  IF newJobFlag = 1
  THEN
    select cz_end_audit (jobID, 'SUCCESS') into rtnCd;
  END IF;

	
END;
		   
 
$$;


ALTER FUNCTION tm_cz.i2b2_rna_seq_zscore_calc(trial_id character varying, partition_name character varying, partition_indx character varying, partitionid numeric, run_type character varying, currentjobid numeric, data_type character varying, log_base bigint, source_cd character varying) OWNER TO tm_cz;

--
-- Name: i2b2_secure_study(text, bigint); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_secure_study(trial_id text, currentjobid bigint DEFAULT NULL::bigint) RETURNS void
    LANGUAGE plpgsql
    AS $_$
DECLARE

/*************************************************************************
* Copyright 2008-2012 Janssen Research d, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/

	--Audit variables
	newJobFlag integer(1);
	databaseName varchar(100);
	procedureName varchar(100);
	jobID bigint;
	stepCt bigint;
	
	v_bio_experiment_id	bigint;
	pExists				integer;
	TrialId				varchar(100);


BEGIN

	TrialId := trial_id;
	
	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;

	PERFORM sys_context('USERENV', 'CURRENT_SCHEMA') INTO databaseName ;
	procedureName := $$PLSQL_UNIT;

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it
	IF(coalesce(jobID::text, '') = '' or jobID < 1)
	THEN
		newJobFlag := 1; -- True
		cz_start_audit (procedureName, databaseName, jobID);
	END IF;
  
	stepCt := 0;
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Start ' || procedureName,0,stepCt,'Done');
	
	--	create security records in observation_fact
	
	i2b2_create_security_for_trial(TrialId, 'Y', jobID);
	
	--	load i2b2_secure
	
	i2b2_load_security_data(jobID);
	
	--	check if entry exists for study in bio_experiment
	
	select count(*) into pExists
	from biomart.bio_experiment
	where accession = TrialId;
	
	if pExists = 0 then
		insert into biomart.bio_experiment
		(title, accession, etl_id)
		PERFORM 'Metadata not available'
			  ,TrialId
			  ,'METADATA:' || TrialId
		;
	    stepCt := stepCt + 1;
		cz_write_audit(jobId,databaseName,procedureName,'Insert trial/study into biomart.bio_experiment',SQL%ROWCOUNT,stepCt,'Done');
		commit;
	end if;
	
	select bio_experiment_id into v_bio_experiment_id
	from biomart.bio_experiment
	where accession = TrialId;
	
	insert into searchapp.search_secure_object
	(bio_data_id
	,display_name
	,data_type
	,bio_data_unique_id
	)
	PERFORM v_bio_experiment_id
	      ,parse_nth_value(md.c_fullname,2,'\') || ' - ' || md.c_name as display_name
		  ,'BIO_CLINICAL_TRIAL' as data_type
		  ,'EXP:' || TrialId as bio_data_unique_id
	from i2b2metadata.i2b2 md
	where md.sourcesystem_cd = TrialId
	  and md.c_hlevel = 
		 (select min(x.c_hlevel) from i2b2metadata.i2b2 x
		  where x.sourcesystem_cd = TrialId)
	  and not exists
		 (select 1 from searchapp.search_secure_object so
		  where v_bio_experiment_id = so.bio_data_id);
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Inserted trial/study into SEARCHAPP search_secure_object',SQL%ROWCOUNT,stepCt,'Done');
	commit;
		
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'End ' || procedureName,SQL%ROWCOUNT,stepCt,'Done');
	commit;
	
    ---Cleanup OVERALL JOB if this proc is being run standalone
	IF newJobFlag = 1
	THEN
		cz_end_audit (jobID, 'SUCCESS');
	END IF;

	EXCEPTION
	WHEN OTHERS THEN
		--Handle errors.
		cz_error_handler (jobID, procedureName);
		
		--End Proc
		cz_end_audit (jobID, 'FAIL');
	
END;
 
$_$;


ALTER FUNCTION tm_cz.i2b2_secure_study(trial_id text, currentjobid bigint) OWNER TO tm_cz;

--
-- Name: i2b2_show_node(character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_show_node(path character varying) RETURNS void
    LANGUAGE plpgsql
    AS $$
BEGIN

  -------------------------------------------------------------
  -- Shows a tree node in I2b2
  -- KCR@20090519 - First Rev
  -------------------------------------------------------------
  if path != ''  or path != '%'
  then

      --I2B2
    UPDATE i2b2
      SET c_visualattributes = 'FA'
    WHERE c_visualattributes LIKE 'F%'
      AND C_FULLNAME LIKE PATH || '%';

     UPDATE i2b2
      SET c_visualattributes = 'LA'
    WHERE c_visualattributes LIKE 'L%'
      AND C_FULLNAME LIKE PATH || '%';
    COMMIT;
  END IF;

END;
 
$$;


ALTER FUNCTION tm_cz.i2b2_show_node(path character varying) OWNER TO tm_cz;

--
-- Name: i2b2_table_bkp(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_table_bkp() RETURNS void
    LANGUAGE plpgsql
    AS $$
DECLARE


	tExists		integer;

BEGIN
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
  
	select count(*) into tExists
	from all_tables
	where owner = 'I2B2METADATA'
      and table_name = 'I2B2_BKP';
	
	if tExists > 0 then
		EXECUTE('drop table i2b2metadata.i2b2_bkp');
	end if;
	
	select count(*) into tExists
	from all_tables
	where owner = 'I2B2DEMODATA'
      and table_name = 'CONCEPT_COUNTS_BKP';
	
	if tExists > 0 then	
		EXECUTE('drop table i2b2demodata.concept_counts_bkp');
	end if;
  
	select count(*) into tExists
	from all_tables
	where owner = 'I2B2DEMODATA'
      and table_name = 'CONCEPT_DIMENSION_BKP';
	
	if tExists > 0 then
		EXECUTE('drop table i2b2demodata.concept_dimension_bkp');
	end if;
  
 	select count(*) into tExists
	from all_tables
	where owner = 'I2B2DEMODATA'
      and table_name = 'OBSERVATION_FACT_BKP';
	
	if tExists > 0 then
		EXECUTE('drop table i2b2demodata.observation_fact_bkp');
	end if;
  
 	select count(*) into tExists
	from all_tables
	where owner = 'I2B2DEMODATA'
      and table_name = 'PATIENT_DIMENSION_BKP';
	
	if tExists > 0 then
		EXECUTE('drop table i2b2demodata.patient_dimension_bkp');
	end if;

	--Backup tables
	EXECUTE 'CREATE TABLE I2B2METADATA.I2B2_BKP AS SELECT * FROM I2B2METADATA.I2B2';
	EXECUTE 'CREATE TABLE I2B2DEMODATA.CONCEPT_COUNTS_BKP AS SELECT * FROM I2B2DEMODATA.CONCEPT_COUNTS';
	EXECUTE 'CREATE TABLE I2B2DEMODATA.CONCEPT_DIMENSION_BKP AS SELECT * FROM I2B2DEMODATA.CONCEPT_DIMENSION';
	EXECUTE 'CREATE TABLE I2B2DEMODATA.OBSERVATION_FACT_BKP AS SELECT * FROM I2B2DEMODATA.OBSERVATION_FACT';
	EXECUTE 'CREATE TABLE I2B2DEMODATA.PATIENT_DIMENSION_BKP AS SELECT * FROM I2B2DEMODATA.PATIENT_DIMENSION';
END;


 
$$;


ALTER FUNCTION tm_cz.i2b2_table_bkp() OWNER TO tm_cz;

--
-- Name: i2b2_table_defrag(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_table_defrag() RETURNS void
    LANGUAGE plpgsql
    AS $$
BEGIN

  -------------------------------------------------------------
  -- Moves the I2B2 tables to reduce defragmentation
  -- KCR@20090527 - First Rev
  -- JEA@20090923 - Removed I2B2DEMODATA.IDX_OB_FACT_3, Oracle doesn't need to index every column like SQL Server (per Aaron A.)
  -------------------------------------------------------------
  EXECUTE 'ALTER TABLE I2B2METADATA.I2B2 MOVE';
  EXECUTE 'ALTER TABLE I2B2DEMODATA.CONCEPT_COUNTS MOVE';
  EXECUTE 'ALTER TABLE I2B2DEMODATA.CONCEPT_DIMENSION MOVE';
  EXECUTE 'ALTER TABLE I2B2DEMODATA.OBSERVATION_FACT MOVE';
  EXECUTE 'ALTER TABLE I2B2DEMODATA.PATIENT_DIMENSION MOVE';
  --Rebuild Indexes
  EXECUTE 'ALTER INDEX I2B2DEMODATA.OB_FACT_PK REBUILD';
  EXECUTE 'ALTER INDEX I2B2DEMODATA.IDX_OB_FACT_1 REBUILD';
  EXECUTE 'ALTER INDEX I2B2DEMODATA.IDX_OB_FACT_2 REBUILD';

  EXECUTE 'ALTER INDEX I2B2DEMODATA.IDX_CONCEPT_DIM_1 REBUILD';
  EXECUTE 'ALTER INDEX I2B2DEMODATA.IDX_CONCEPT_DIM_2 REBUILD';

  EXECUTE 'ALTER INDEX I2B2METADATA.IDX_I2B2_A REBUILD';
  EXECUTE 'ALTER INDEX I2B2METADATA.IDX_I2B2_B REBUILD';

  EXECUTE 'ALTER INDEX I2B2DEMODATA.CONCEPT_COUNTS_INDEX1 REBUILD';

END;






 
$$;


ALTER FUNCTION tm_cz.i2b2_table_defrag() OWNER TO tm_cz;

--
-- Name: i2b2_truncate_release_tables(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.i2b2_truncate_release_tables() RETURNS void
    LANGUAGE plpgsql
    AS $_$
DECLARE

	--	Procedure to run one test in CZ_TEST

	--	JEA@20111019	New

	--	Define the abstract result set record

	TYPE r_type IS RECORD (
		rtn_text          varchar(2000)
	);

	--	Define the abstract result set table
	TYPE tr_type IS TABLE OF r_type;

	--	Define the result set

	rtn_array tr_type;

	--	Variables

	tText 			varchar(2000);

    --Audit variables
	newJobFlag integer(1);
	databaseName varchar(100);
	procedureName varchar(100);
	jobID bigint;
	stepCt bigint;

	
BEGIN

	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := -1;

	PERFORM sys_context('USERENV', 'CURRENT_SCHEMA') INTO databaseName ;
	procedureName := $$PLSQL_UNIT;

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it
	IF(coalesce(jobID::text, '') = '' or jobID < 1)
	THEN
		newJobFlag := 1; -- True
		cz_start_audit (procedureName, databaseName, jobID);
	END IF;

	stepCt := 0;
	cz_write_audit(jobId,databaseName,procedureName,'Starting i2b2_truncate_release_tablese',0,stepCt,'Done');
	stepCt := stepCt + 1;

	tText := 'Select table_name from all_tables where owner = ' || '''' || 'TM_CZ' || '''' || 'and table_name like ' || '''' || '%_RELEASE' || '''';

	EXECUTE(tText) BULK COLLECT INTO rtn_array;

	for i in rtn_array.first .. rtn_array.last
	loop
		RAISE NOTICE '%', rtn_array(i).rtn_text;

		if (rtn_array(i)(.rtn_text IS NOT NULL AND .rtn_text::text <> '')) then
			tText := 'truncate table ' || rtn_array(i).rtn_text;

			EXECUTE(tText);
			tText := 'Truncated ' || rtn_array(i).rtn_text;

			cz_write_audit(jobId,databaseName,procedureName,tText,0,stepCt,'Done');

		end if;

	end loop;

	cz_write_audit(jobId,databaseName,procedureName,'End i2b2_truncate_release_tablese',0,stepCt,'Done');
	stepCt := stepCt + 1;

END;
 
$_$;


ALTER FUNCTION tm_cz.i2b2_truncate_release_tables() OWNER TO tm_cz;

--
-- Name: instr(character varying, character varying, integer, integer); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.instr(string character varying, string_to_search character varying, beg_index integer DEFAULT 1, occur_index integer DEFAULT 1) RETURNS integer
    LANGUAGE plpgsql IMMUTABLE STRICT
    AS $$
DECLARE
    pos integer NOT NULL DEFAULT 0;
    occur_number integer NOT NULL DEFAULT 0;
    temp_str varchar;
    beg integer;
    i integer;
    length integer;
    ss_length integer;
BEGIN
    IF beg_index > 0 THEN
        beg := beg_index;
        temp_str := substring(string FROM beg_index);

        FOR i IN 1..occur_index LOOP
            pos := position(string_to_search IN temp_str);

            IF i = 1 THEN
                beg := beg + pos - 1;
            ELSE
                beg := beg + pos;
            END IF;

            temp_str := substring(string FROM beg + 1);
        END LOOP;

        IF pos = 0 THEN
            RETURN 0;
        ELSE
            RETURN beg;
        END IF;
    ELSE
        ss_length := char_length(string_to_search);
        length := char_length(string);
        beg := length + beg_index - ss_length + 2;

        WHILE beg > 0 LOOP
            temp_str := substring(string FROM beg FOR ss_length);
            pos := position(string_to_search IN temp_str);

            IF pos > 0 THEN
                occur_number := occur_number + 1;

                IF occur_number = occur_index THEN
                    RETURN beg;
                END IF;
            END IF;

            beg := beg - 1;
        END LOOP;

        RETURN 0;
    END IF;
END;
$$;


ALTER FUNCTION tm_cz.instr(string character varying, string_to_search character varying, beg_index integer, occur_index integer) OWNER TO tm_cz;

--
-- Name: is_date(character varying, character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.is_date(character varying, character varying) RETURNS numeric
    LANGUAGE plpgsql IMMUTABLE STRICT
    AS $_$
declare
        i numeric;
begin
        i := $1::date;
        return 0;
        EXCEPTION WHEN invalid_text_representation then
                return 1;
end;
$_$;


ALTER FUNCTION tm_cz.is_date(character varying, character varying) OWNER TO tm_cz;

--
-- Name: is_number_v2(character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.is_number_v2(p_string character varying) RETURNS numeric
    LANGUAGE plpgsql
    AS $$
DECLARE

        l_number numeric;
    
BEGIN
        l_number := p_string;
        return l_number;
   exception
       when others then
           return 1;
   end;

$$;


ALTER FUNCTION tm_cz.is_number_v2(p_string character varying) OWNER TO tm_cz;

--
-- Name: is_numeric(character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.is_numeric(character varying) RETURNS numeric
    LANGUAGE plpgsql IMMUTABLE STRICT
    AS $_$ 
declare 
        i numeric; 
begin 
        i := $1::numeric; 
        return 0; 
        EXCEPTION WHEN invalid_text_representation then 
                return 1; 
end; 
$_$;


ALTER FUNCTION tm_cz.is_numeric(character varying) OWNER TO tm_cz;

--
-- Name: isnumeric(character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.isnumeric(p_string character varying) RETURNS numeric
    LANGUAGE plpgsql
    AS $$
DECLARE

      l_number numeric;
  
BEGIN
      l_number := p_string;
      return 1;
  exception
      when others then
          return 0;
  end;

$$;


ALTER FUNCTION tm_cz.isnumeric(p_string character varying) OWNER TO tm_cz;

--
-- Name: jnj_init_cap(text); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.jnj_init_cap(text_to_parse text) RETURNS character varying
    LANGUAGE plpgsql
    AS $$
DECLARE

   start_pos 		bigint;
   end_pos   		bigint;
   string_length 	integer;
   string_tokens 	varchar(32676);
   counter 			integer;
   token_value 		varchar(1000);
   text_delimiter 	char(1);
   noInitCap 		boolean;
   
   --	create array to hold strings that will not be initcapped
   
	type excluded_aat is table of category_path_excluded_words%ROWTYPE index by integer;
	excludedText excluded_aat;
	exclCt integer;
   
   --	text to return
   initcap_text varchar(1000);
   

BEGIN
  -------------------------------------------------------------------------------
   -- Performs custom initcap for category paths where specific text strings are 
   -- excluded from the process.  Strings are delimited by a space.  The \ in
   -- the category path are converted to ' \ ' before parsing.
   
   -- JEA@20091001 - First rev. 
   -- Copyright ? 2009 Recombinant Data Corp.
   -------------------------------------------------------------------------------

	--	Load exclusion text
  
	select excluded_text
		bulk collect into excludedText
		from category_path_excluded_words;
  
	--	Add a delimiter to the end of the string so we dont lose last value and
	--	surround \ with spaces
	
	text_delimiter := ' ';
	string_tokens := replace(text_to_parse,'\',' \ ') || text_delimiter;
   
	--get length of string
	string_length := length(string_tokens);

	--set start and end for first token
	start_pos := 1;
	end_pos   := instr(string_tokens,text_delimiter,1,1);
	counter := 1;

	LOOP
		--	Get substring
		token_value := substr(string_tokens, start_pos, end_pos - start_pos);
	
		--	check if token_value is in excludedText, if yes, set indicator
	
		noInitCap := false;
		exclCt := excludedText.FIRST;

		while ((exclCt IS NOT NULL AND exclCt::text <> '') and not noInitCap)
		loop
			if token_value = excludedText(exclCt).excluded_text then
				noInitCap := true;
			end if;
			exclCt := excludedText.NEXT (exclCt);
		end loop;
	
		if noInitCap then
			initcap_text := initcap_text || token_value || ' ';
		else
			initcap_text := initcap_text || initcap(token_value) || ' ';
		end if;
    
		--Check to see if we are done
		IF end_pos = string_length  
		THEN
			initcap_text := replace(rtrim(initcap_text,' '),' \ ','\');
			EXIT;
		ELSE  
			-- Increment Start Pos and End Pos
			start_pos := end_pos + 1;
			--	increment counter
			counter := counter + 1;
			end_pos := instr(string_tokens, text_delimiter,1, counter);
      
		END IF;
  END LOOP;
  
  return initcap_text;
  
END JNJ_INIT_CAP;
 
$$;


ALTER FUNCTION tm_cz.jnj_init_cap(text_to_parse text) OWNER TO tm_cz;

--
-- Name: load_kegg_content_data(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.load_kegg_content_data() RETURNS void
    LANGUAGE plpgsql
    AS $$
BEGIN

begin

  delete from bio_content_reference
    where bio_content_id in
      (select bio_file_content_id
      from bio_content
      where repository_id in
        (select bio_content_repo_id
        from bio_content_repository
        where upper(repository_type)='KEGG')
      );
  --806
  delete from bio_content
    where repository_id =
      (select bio_content_repo_id
      from bio_content_repository
      where upper(repository_type)='KEGG');
  --806
  delete from bio_content_repository
    where upper(repository_type)='KEGG';
  --1
commit;
end;

begin
-- populate bio_content_repository
  insert into bio_content_repository(
    location
  , active_y_n
  , repository_type
  , location_type
  )
  values (
    'http://www.genome.jp/'
  , 'Y'
  , 'Kegg'
  , 'URL'
  );
commit;
end;

begin

  insert into bio_content(
  --  file_name
    repository_id
  , location
  --, title  , abstract
  , file_type
  --, etl_id
  )
  PERFORM distinct
    bcr.bio_content_repo_id
  , bcr.location||'dbget-bin/show_pathway?'|| bm.primary_external_id
  , 'Data'
  from
    bio_content_repository bcr
  , bio_marker bm
  where upper(bcr.repository_type)='KEGG'
  and upper(bm.primary_source_code)='KEGG';
  --806 rows inserted
commit;
end;

begin

  insert into bio_content_reference(
    bio_content_id
  , bio_data_id
  , content_reference_type
  )
  PERFORM distinct
    bc.bio_file_content_id
  , path.bio_marker_id
  , bcr.location_type
  from
    bio_content bc
  , bio_marker path
  , bio_content_repository bcr
  where bc.repository_id = bcr.bio_content_repo_id
  and path.primary_external_id=substring(bc.location from length(bc.location)-7)
  and path.primary_source_code='KEGG';
  --806
commit;
end;


end;
 
$$;


ALTER FUNCTION tm_cz.load_kegg_content_data() OWNER TO tm_cz;

--
-- Name: num_occurances(character varying, character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.num_occurances(input_str character varying, search_str character varying) RETURNS integer
    LANGUAGE plpgsql
    AS $$
DECLARE

  num integer;

BEGIN
  num := 0;
  while instr(input_str, search_str, 1, num + 1) > 0 loop
    num := num + 1;
  end loop;
  return num;
end;

$$;


ALTER FUNCTION tm_cz.num_occurances(input_str character varying, search_str character varying) OWNER TO tm_cz;

--
-- Name: number_parser(character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.number_parser(numbers_to_parse character varying) RETURNS numeric[]
    LANGUAGE plpgsql
    AS $$
DECLARE

   start_pos bigint;
   end_pos   bigint;
   string_length integer;
   string_tokens varchar(32676);
   counter integer;
   token_value varchar(32676);

   list_values _numeric;


BEGIN
  -------------------------------------------------------------------------------
   -- Populates a temp_token table with parsed values for any comma separated list.
   -- Requires a type so that multiple records can exist for different uses.
   -- KCR@20090106 - First rev.
   -- Copyright c 2009 Recombinant Data Corp.
   -------------------------------------------------------------------------------

  --Add a delimiter to the end of the string so we dont lose last value
  string_tokens := numbers_to_parse || ',';

  --Initialize the collection
  list_values := NUMBER_TABLE() ;

  --get length of string
  string_length := length(string_tokens);

  --set start and end for first token
  start_pos := 1;
  end_pos   := instr(string_tokens,',',1,1);
  counter := 1;
  LOOP
    --Get substring
    token_value := to_number(substr(string_tokens, start_pos, end_pos - start_pos));

    --add values to collection
   list_values.EXTEND;
   list_values(list_Values.LAST):= token_value;

    --Check to see if we are done
    IF end_pos = string_length
    THEN
      EXIT;
    ELSE
      -- Increment Start Pos and End Pos
      start_pos := end_pos + 1;
      --increment counter
      counter := counter + 1;
      end_pos := instr(string_tokens, ',',1, counter);

    END IF;

  END LOOP;

  return list_values;
  --on an invalid value (Can't convert to number, just return the table of numbers.
  EXCEPTION WHEN OTHERS THEN
    return list_values;


END number_parser;
 
$$;


ALTER FUNCTION tm_cz.number_parser(numbers_to_parse character varying) OWNER TO tm_cz;

--
-- Name: nz_i2b2_copy_to_stage(character varying, character varying, bigint); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.nz_i2b2_copy_to_stage(character varying, character varying, bigint) RETURNS integer
    LANGUAGE plpgsql IMMUTABLE SECURITY DEFINER
    AS $_$
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
Declare
	--	Alias for parameters
	
	trial_id  alias for $1;
	data_type alias for $2;
	currentJobID alias for $3;
	rtnCd integer;
	TrialId 	varchar(200);
	msgText		varchar(2000);
	dataType	varchar(50);

	tText			varchar(2000);
	tExists 		integer;
	tCount			integer;
	source_table	varchar(50);
	release_table	varchar(50);
	vSNP 			integer;
	rowCt			bigint;
	v_sqlerrm		varchar(1000);

	--Audit variables
	newJobFlag integer;
	databaseName VARCHAR(100);
	procedureName VARCHAR(100);
	jobID numeric(18,0);
	stepCt numeric(18,0);
  
	r_stage_table	record;

	
BEGIN

	TrialID := upper(trial_id);
	dataType := upper(data_type);
	
	stepCt := 0;
	tCount := 0;
	
	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;

	databaseName := 'TM_CZ';
	procedureName := 'I2B2_COPY_TO_STAGE';

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it
	IF(jobID IS NULL or jobID < 1)
	THEN
		newJobFlag := 1; -- True
		jobId := tm_cz.czx_start_audit (procedureName, databaseName);
	END IF;
  
	stepCt := stepCt + 1;
	select tm_cz.czx_write_audit(jobId,databaseName,procedureName,'Starting ' || procedureName,0,stepCt,'Done')into rtnCd;
	
	stepCt := stepCt + 1;
	msgText := 'Extracting trial: ' || TrialId;
	select tm_cz.czx_write_audit(jobId,databaseName,procedureName, msgText,0,stepCt,'Done')into rtnCd;

	if TrialId = null then
		stepCt := stepCt + 1;
		select tm_cz.czx_write_audit(jobId,databaseName,procedureName,'TrialID missing',0,stepCt,'Done')into rtnCd;
		Return 16;
	end if;
	
	for r_stage_table in
		select upper(table_owner) as table_owner
			  ,upper(table_name) as table_name
			  ,upper(study_specific) as study_specific
			  ,where_clause
			  ,upper(stage_table_name) as stage_table_name
		from tm_cz.migrate_tables
		where instr(dataType,data_type) > 0
	loop
		source_table := r_stage_table.table_owner || '.' || r_stage_table.table_name;
		release_table := 'tm_stage.' || r_stage_table.stage_table_name;
		stepCt := stepCt + 1;
		select tm_cz.czx_write_audit(jobId,databaseName,procedureName,'Processing ' || source_table,0,stepCt,'Done')into rtnCd;
		tCount := tCount + 1;
		
		if r_stage_table.study_specific = 'Y' then
			tText := 'delete from ' || release_table || ' where release_study = ' || '''' || TrialId || '''';
			execute immediate tText;
			rowCt := ROW_COUNT;
			stepCt := stepCt + 1;
			select tm_cz.czx_write_audit(jobId,databaseName,procedureName,'Deleted study from ' || release_table,rowCt,stepCt,'Done')into rtnCd;
			
			tText := 'insert into ' || release_table || ' select st.*,' || '''' || TrialId || '''' || ' from ' || source_table || ' st ' || 
					 replace(r_stage_table.where_clause,'TrialId','''' || TrialId || '''');
			execute immediate tText ;
			rowCt := ROW_COUNT;
			stepCt := stepCt + 1;
			select tm_cz.czx_write_audit(jobId,databaseName,procedureName,'Inserted study into ' || release_table,rowCt,stepCt,'Done')into rtnCd;
		else
			tText := 'truncate table ' || release_table;
			stepCt := stepCt + 1;		
			execute immediate tText;
			select tm_cz.czx_write_audit(jobId,databaseName,procedureName,'Truncated '|| release_table,0,stepCt,'Done')into rtnCd;
			tText := 'insert into ' || release_table || ' select st.* from ' || source_table || ' st ';
			execute immediate(tText);
			rowCt := ROW_COUNT;
			stepCt := stepCt + 1;
			select tm_cz.czx_write_audit(jobId,databaseName,procedureName,'Inserted all data into ' || release_table,rowCt,stepCt,'Done')into rtnCd;
		end if;
			
	end loop;
	
	if tCount = 0 then
		stepCt := stepCt + 1;
		select tm_cz.czx_write_audit(jobId,databaseName,procedureName,'data_type invalid: '|| data_type,0,stepCt,'Done')into rtnCd;
		Return 16;
	end if;
	stepCt := stepCt + 1;
	select tm_cz.czx_write_audit(jobId,databaseName,procedureName,'End '||procedureName,0,stepCt,'Done')into rtnCd;

       ---Cleanup OVERALL JOB if this proc is being run standalone
  IF newJobFlag = 1
  THEN
    select tm_cz.czx_end_audit (jobID, 'SUCCESS')into rtnCd;
  END IF;
  
  return 0;

  EXCEPTION
  WHEN OTHERS THEN
	v_sqlerrm := substr(SQLERRM,1,1000);
	raise notice 'error: %', v_sqlerrm;
    --Handle errors.
    select tm_cz.czx_error_handler (jobID, procedureName,v_sqlerrm)into rtnCd;
    --End Proc
    select tm_cz.czx_end_audit (jobID, 'FAIL')into rtnCd;
	return 16;
END;
$_$;


ALTER FUNCTION tm_cz.nz_i2b2_copy_to_stage(character varying, character varying, bigint) OWNER TO tm_cz;

--
-- Name: parse_nth_value(character varying, numeric, character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.parse_nth_value(pvalue character varying, location numeric, delimiter character varying) RETURNS character varying
    LANGUAGE plpgsql
    AS $$
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
Declare
   v_posA integer;
   v_posB integer;
   iLoc	  integer;

begin
   iLoc := location;
   if location = 1 then
      v_posA := 1; -- Start at the beginning
   else
	  select tm_cz.instr(pValue,delimiter, 1, iLoc-1) into v_posA;
 --     v_posA := tm_cz.instr(pValue,delimiter, 1, location - 1); 
      if v_posA = 0 then
         return null; --No values left.
      end if;
      v_posA := v_posA + length(delimiter);
   end if;

   select tm_cz.instr (pValue, delimiter, 1, iLoc) into v_posB;
   --v_posB := tm_cz.instr (pValue, delimiter, 1, location);
   if v_posB = 0 then -- Use the end of the file
      return substr (pValue, v_posA);
   end if;
   
   return substr (pValue, v_posA, v_posB - v_posA);

end;
$$;


ALTER FUNCTION tm_cz.parse_nth_value(pvalue character varying, location numeric, delimiter character varying) OWNER TO tm_cz;

--
-- Name: rdc_init_cap(text); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.rdc_init_cap(text_to_parse text) RETURNS character varying
    LANGUAGE plpgsql
    AS $$
DECLARE

   start_pos 		bigint;
   end_pos   		bigint;
   string_length 	integer;
   string_tokens 	varchar(32676);
   counter 			integer;
   token_value 		varchar(1000);
   text_delimiter 	char(1);
   noInitCap 		boolean;

   --	create array to hold strings that will not be initcapped

	type excluded_aat is table of category_path_excluded_words%ROWTYPE index by integer;
	excludedText excluded_aat;
	exclCt integer;

   --	text to return
   initcap_text varchar(1000);


BEGIN
  -------------------------------------------------------------------------------
   -- Performs custom initcap for category paths where specific text strings are
   -- excluded from the process.  Strings are delimited by a space.  The \ in
   -- the category path are converted to ' \ ' before parsing.

   -- JEA@20091001 - First rev.
   -- Copyright ? 2009 Recombinant Data Corp.
   -------------------------------------------------------------------------------

	--	Load exclusion text

	select excluded_text
		bulk collect into excludedText
		from category_path_excluded_words;

	--	Add a delimiter to the end of the string so we dont lose last value and
	--	surround \ with spaces

	text_delimiter := ' ';
	string_tokens := replace(text_to_parse,'\',' \ ') || text_delimiter;

	--get length of string
	string_length := length(string_tokens);

	--set start and end for first token
	start_pos := 1;
	end_pos   := instr(string_tokens,text_delimiter,1,1);
	counter := 1;

	LOOP
		--	Get substring
		token_value := substr(string_tokens, start_pos, end_pos - start_pos);

		--	check if token_value is in excludedText, if yes, set indicator

		noInitCap := false;
		exclCt := excludedText.FIRST;

		if (substring(token_value from 1 for 1) = '(' and substring(token_value from length(token_value) for 1) = ')' ) then
			noInitCap := true;
		else
		while ((exclCt IS NOT NULL AND exclCt::text <> '') and not noInitCap)
			loop
				if token_value = excludedText(exclCt).excluded_text then
					noInitCap := true;
				end if;
				exclCt := excludedText.NEXT (exclCt);
			end loop;
		end if;

		if noInitCap then
			initcap_text := initcap_text || token_value || ' ';
		else
			initcap_text := initcap_text || initcap(token_value) || ' ';
		end if;

		--Check to see if we are done
		IF end_pos = string_length
		THEN
			initcap_text := replace(rtrim(initcap_text,' '),' \ ','\');
			EXIT;
		ELSE
			-- Increment Start Pos and End Pos
			start_pos := end_pos + 1;
			--	increment counter
			counter := counter + 1;
			end_pos := instr(string_tokens, text_delimiter,1, counter);

		END IF;
  END LOOP;

  return initcap_text;

END RDC_INIT_CAP;
 
$$;


ALTER FUNCTION tm_cz.rdc_init_cap(text_to_parse text) OWNER TO tm_cz;

--
-- Name: rdc_reload_mrna_data(text, text, text, bigint, bigint); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.rdc_reload_mrna_data(trial_id text, data_type text DEFAULT 'R'::text, source_cd text DEFAULT 'STD'::text, log_base bigint DEFAULT 2, currentjobid bigint DEFAULT NULL::bigint) RETURNS void
    LANGUAGE plpgsql
    AS $_$
DECLARE


	--	JEA@20111202	One-off to re-zscore gene expression data for a study

  TrialID		varchar(100);
  RootNode		varchar(2000);
  root_level	integer;
  topNode		varchar(2000);
  topLevel		integer;
  tPath			varchar(2000);
  study_name	varchar(100);
  sourceCd		varchar(50);

  dataType		varchar(10);
  sqlText		varchar(1000);
  tText			varchar(1000);
  gplTitle		varchar(1000);
  pExists		bigint;
  partTbl   	bigint;
  partExists 	bigint;
  sampleCt		bigint;
  idxExists 	bigint;
  logBase		bigint;
  pCount		integer;
  sCount		integer;
  tablespaceName	varchar(200);
  
    --Audit variables
  newJobFlag integer(1);
  databaseName varchar(100);
  procedureName varchar(100);
  jobID bigint;
  stepCt bigint;
  
  --unmapped_patients exception;
  missing_platform	exception;
  missing_tissue	EXCEPTION;
  unmapped_platform exception;
  


BEGIN
	TrialID := upper(trial_id);
--	topNode := REGEXP_REPLACE('\' || top_node || '\','(\\){2,}', '\');	
--	select length(topNode)-length(replace(topNode,'\','')) into topLevel from dual;
	
	if coalesce(data_type::text, '') = '' then
		dataType := 'R';
	else
		if data_type in ('R','T','L') then
			dataType := data_type;
		else
			dataType := 'R';
		end if;
	end if;
	
	logBase := log_base;
	sourceCd := upper(coalesce(source_cd,'STD'));

  --Set Audit Parameters
  newJobFlag := 0; -- False (Default)
  jobID := currentJobID;

  PERFORM sys_context('USERENV', 'CURRENT_SCHEMA') INTO databaseName ;
  procedureName := $$PLSQL_UNIT;

  --Audit JOB Initialization
  --If Job ID does not exist, then this is a single procedure run and we need to create it
  IF(coalesce(jobID::text, '') = '' or jobID < 1)
  THEN
    newJobFlag := 1; -- True
    cz_start_audit (procedureName, databaseName, jobID);
  END IF;
    	
	stepCt := 0;
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Starting i2b2_process_mrna_data',0,stepCt,'Done');
		
	--	truncate tmp tables

	EXECUTE('truncate table tm_wz.wt_subject_microarray_logs');
	EXECUTE('truncate table tm_wz.wt_subject_microarray_calcs');
	EXECUTE('truncate table tm_wz.wt_subject_microarray_med');
	
	select count(*) 
	into idxExists
	from all_indexes
	where table_name = 'WT_SUBJECT_MICROARRAY_LOGS'
	  and index_name = 'WT_SUBJECT_MRNA_LOGS_I1'
	  and owner = 'TM_WZ';
		
	if idxExists = 1 then
		EXECUTE('drop index tm_wz.wt_subject_mrna_logs_i1');		
	end if;
	
	select count(*) 
	into idxExists
	from all_indexes
	where table_name = 'WT_SUBJECT_MICROARRAY_CALCS'
	  and index_name = 'WT_SUBJECT_MRNA_CALCS_I1'
	  and owner = 'TM_WZ';
		
	if idxExists = 1 then
		EXECUTE('drop index tm_wz.wt_subject_mrna_calcs_i1');
	end if;
	
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Truncate work tables in TM_WZ',0,stepCt,'Done');
	
	insert into wt_subject_microarray_logs 
	(probeset_id
	,intensity_value
	,assay_id
	,log_intensity
	,patient_id
	,sample_id
	,subject_id
	)
	PERFORM distinct gs.probeset_id
		  ,avg(md.intensity_value)
		  ,sd.assay_id
		  ,avg(md.intensity_value)
		  ,sd.patient_id
		  ,sd.sample_id
		  ,sd.subject_id
	from de_subject_sample_mapping sd
		,lz_src_mrna_data md   
		,de_mrna_annotation gs
	where sd.sample_cd = md.expr_id
	  and sd.platform = 'MRNA_AFFYMETRIX'
	  and sd.trial_name = TrialId
	  and md.trial_name = TrialId
	  and sd.gpl_id = gs.gpl_id
	  and md.probeset = gs.probe_id
	  group by gs.probeset_id
		  ,sd.assay_id
		  ,sd.patient_id
		  ,sd.sample_id
		  ,sd.subject_id;
	
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Loaded data for trial in TM_WZ wt_subject_microarray_logs',SQL%ROWCOUNT,stepCt,'Done');

	commit;
    
	EXECUTE('create index tm_wz.wt_subject_mrna_logs_i1 on tm_wz.wt_subject_microarray_logs (trial_name, probeset_id) nologging  tablespace "INDX"');
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Create index on TM_WZ wt_subject_microarray_logs',0,stepCt,'Done');
		
--	calculate mean_intensity, median_intensity, and stddev_intensity per experiment, probe

	insert into wt_subject_microarray_calcs
	(trial_name
	,probeset_id
	,mean_intensity
	,median_intensity
	,stddev_intensity
	)
	PERFORM d.trial_name 
		  ,d.probeset_id
		  ,avg(log_intensity)
		  ,median(log_intensity)
		  ,stddev(log_intensity)
	from wt_subject_microarray_logs d 
	group by d.trial_name 
			,d.probeset_id;
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Calculate intensities for trial in TM_WZ wt_subject_microarray_calcs',SQL%ROWCOUNT,stepCt,'Done');

	commit;

	EXECUTE('create index tm_wz.wt_subject_mrna_calcs_i1 on tm_wz.wt_subject_microarray_calcs (trial_name, probeset_id) nologging tablespace "INDX"');
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Create index on TM_WZ wt_subject_microarray_calcs',0,stepCt,'Done');
		
-- calculate zscore

	insert into wt_subject_microarray_med parallel 
	(probeset_id
	,intensity_value
	,log_intensity
	,assay_id
	,mean_intensity
	,stddev_intensity
	,median_intensity
	,zscore
	,patient_id
	,sample_id
	,subject_id)
	PERFORM d.probeset_id
		  ,d.intensity_value 
		  ,d.log_intensity 
		  ,d.assay_id  
		  ,c.mean_intensity 
		  ,c.stddev_intensity 
		  ,c.median_intensity 
		  ,CASE WHEN stddev_intensity=0 THEN 0 ELSE (log_intensity - median_intensity ) / stddev_intensity END
		  ,d.patient_id
		  ,d.sample_id
		  ,d.subject_id
    from wt_subject_microarray_logs d 
		,wt_subject_microarray_calcs c 
    where d.probeset_id = c.probeset_id;
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Calculate Z-Score for trial in TM_WZ wt_subject_microarray_med',SQL%ROWCOUNT,stepCt,'Done');

    commit;

	select count(*) 
	into idxExists
	from all_indexes
	where table_name = 'DE_SUBJECT_MICROARRAY_DATA'
	  and index_name = 'MRNA_INDX1'
	  and owner = 'DEAPP';
		
	if idxExists = 0 then
		EXECUTE('create index deapp.mrna_idx1 on deapp.de_subject_microarray_data (trial_name) nologging tablespace "INDX"');
	end if;
	
	delete from de_subject_microarray_data
	where trial_name = TrialId;
	
	EXECUTE('drop index deapp.mrna_idx1');

	insert into de_subject_microarray_data
	(trial_name
	,assay_id
	,probeset_id
	,raw_intensity 
	,log_intensity
	,zscore
	,patient_id
	,sample_id
	,subject_id
	)
	PERFORM TrialId
	      ,m.assay_id
	      ,m.probeset_id 
		  ,case when dataType = 'R' then m.intensity_value
		        when dataType = 'L' then case when logBase = -1 then null else power(logBase, m.log_intensity) end
				else null end
		  ,m.log_intensity
	      ,round(CASE WHEN m.zscore < -2.5 THEN -2.5 WHEN m.zscore >  2.5 THEN  2.5 ELSE round(m.zscore,5) END,5)
		  ,m.patient_id
		  ,m.sample_id
		  ,m.subject_id
	from wt_subject_microarray_med m;
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Insert data for trial in DEAPP de_subject_microarray_data',SQL%ROWCOUNT,stepCt,'Done');

  	commit;
	
--	cleanup tmp_ files

	EXECUTE('truncate table tm_wz.wt_subject_microarray_logs');
	EXECUTE('truncate table tm_wz.wt_subject_microarray_calcs');
	EXECUTE('truncate table tm_wz.wt_subject_microarray_med');

   	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Truncate work tables in TM_WZ',0,stepCt,'Done');
	
    ---Cleanup OVERALL JOB if this proc is being run standalone
	
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'End i2b2_process_mrna_data',0,stepCt,'Done');

	IF newJobFlag = 1
	THEN
		cz_end_audit (jobID, 'SUCCESS');
	END IF;
	
	--select 0 into rtn_code from dual;

	EXCEPTION
	--when unmapped_patients then
	--	cz_write_audit(jobId,databasename,procedurename,'No site_id/subject_id mapped to patient_dimension',1,stepCt,'ERROR');
	--	cz_error_handler(jobid,procedurename);
	--	cz_end_audit (jobId,'FAIL');
	when missing_platform then
		cz_write_audit(jobId,databasename,procedurename,'Platform data missing from one or more subject_sample mapping records',1,stepCt,'ERROR');
		cz_error_handler(jobid,procedurename);
		cz_end_audit (jobId,'FAIL');
		--select 16 into rtn_code from dual;
	when missing_tissue then
		cz_write_audit(jobId,databasename,procedurename,'Tissue Type data missing from one or more subject_sample mapping records',1,stepCt,'ERROR');
		cz_error_handler(jobid,procedurename);
		CZ_END_AUDIT (JOBID,'FAIL');
		--select 16 into rtn_code from dual;
	when unmapped_platform then
		cz_write_audit(jobId,databasename,procedurename,'Platform not found in de_mrna_annotation',1,stepCt,'ERROR');
		CZ_ERROR_HANDLER(JOBID,PROCEDURENAME);
		cz_end_audit (jobId,'FAIL');
		--select 16 into rtn_code from dual;
	WHEN OTHERS THEN
		--Handle errors.
		cz_error_handler (jobID, procedureName);
		--End Proc
		cz_end_audit (jobID, 'FAIL');
		--select 16 into rtn_code from dual;
END;
 
$_$;


ALTER FUNCTION tm_cz.rdc_reload_mrna_data(trial_id text, data_type text, source_cd text, log_base bigint, currentjobid bigint) OWNER TO tm_cz;

--
-- Name: rename_program(character varying, character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.rename_program(oldprogramname character varying, newprogramname character varying) RETURNS void
    LANGUAGE plpgsql
    AS $$
DECLARE

oldTopNode		varchar(2000);
newTopNode		varchar(2000);
regex1		varchar(2000);
regex2		varchar(2000);

BEGIN

DBMS_OUTPUT.ENABLE (20000);
  oldTopNode := '\' || oldProgramName || '\' ;
  newTopNode := '\' || newProgramName || '\' ;
  regex1 := '\\' || oldProgramName || '\\' || '(.*)';
  regex2 := '\\' || newProgramName || '\\' || '\1';
  
  update i2b2metadata.i2b2 set c_fullname=REGEXP_REPLACE(c_fullname, regex1, regex2) where c_fullname like oldTopNode||'%';
  update i2b2metadata.i2b2 set c_dimcode=REGEXP_REPLACE(c_dimcode, regex1, regex2) where c_dimcode like oldTopNode||'%';
  update i2b2metadata.i2b2 set c_tooltip=REGEXP_REPLACE(c_tooltip, regex1, regex2)  where c_tooltip like oldTopNode||'%';
  update i2b2metadata.i2b2 set c_name=newProgramName where c_fullname=newTopNode;
  
  update i2b2metadata.i2b2_secure set c_fullname=REGEXP_REPLACE(c_fullname, regex1, regex2) where c_fullname like oldTopNode||'%';
  update i2b2metadata.i2b2_secure set c_dimcode=REGEXP_REPLACE(c_dimcode, regex1, regex2) where c_dimcode like oldTopNode||'%';
  update i2b2metadata.i2b2_secure set c_tooltip=REGEXP_REPLACE(c_tooltip, regex1, regex2) where c_tooltip like oldTopNode||'%';
  update i2b2metadata.i2b2_secure set c_name=newProgramName where c_fullname=newTopNode;
  
  update i2b2metadata.table_access set c_fullname=REGEXP_REPLACE(c_fullname, regex1, regex2) where c_fullname like oldTopNode||'%';
  update i2b2metadata.table_access set c_dimcode=REGEXP_REPLACE(c_dimcode, regex1, regex2) where c_dimcode like oldTopNode||'%';
  update i2b2metadata.table_access set c_tooltip=REGEXP_REPLACE(c_tooltip, regex1, regex2) where c_tooltip like oldTopNode||'%';
  update i2b2metadata.table_access set c_name=newProgramName where c_fullname=newTopNode;
  update i2b2metadata.table_access set c_table_cd=newProgramName where c_fullname=newTopNode;
 
  update I2B2DEMODATA.concept_counts set concept_path=REGEXP_REPLACE(concept_path, regex1, regex2) where concept_path like oldTopNode||'%';
  update I2B2DEMODATA.concept_counts set parent_concept_path=REGEXP_REPLACE(parent_concept_path, regex1, regex2)where parent_concept_path like oldTopNode||'%';
 
  update I2B2DEMODATA.concept_dimension set concept_path=REGEXP_REPLACE(concept_path, regex1, regex2) where concept_path like oldTopNode||'%';

 commit;
 
END;
 
$$;


ALTER FUNCTION tm_cz.rename_program(oldprogramname character varying, newprogramname character varying) OWNER TO tm_cz;

--
-- Name: rename_study(character varying, character varying, character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.rename_study(programname character varying, oldtitle character varying, newtitle character varying) RETURNS void
    LANGUAGE plpgsql
    AS $$
DECLARE

oldTopNode		varchar(2000);
newTopNode		varchar(2000);
regex1		varchar(2000);
regex2		varchar(2000);

BEGIN

DBMS_OUTPUT.ENABLE (20000);
  oldTopNode := '\' || programName || '\' || oldTitle|| '\';
  newTopNode := '\' || programName || '\' || newTitle|| '\';
  regex1 := '\\' || replace(replace(programName, '(', '\('), ')', '\)') || '\\' || replace(replace(oldTitle, '(', '\('), ')', '\)')|| '\\' || '(.*)';
  regex2 := '\\' || programName || '\\' || newTitle|| '\\' || '\1';
  
  update i2b2metadata.i2b2 set c_fullname=REGEXP_REPLACE(c_fullname, regex1, regex2);
  update i2b2metadata.i2b2 set c_dimcode=REGEXP_REPLACE(c_dimcode, regex1, regex2);
  update i2b2metadata.i2b2 set c_tooltip=REGEXP_REPLACE(c_tooltip, regex1, regex2);
  update i2b2metadata.i2b2 set c_name=newTitle where c_fullname=newTopNode;
  
  update i2b2metadata.i2b2_secure set c_fullname=REGEXP_REPLACE(c_fullname, regex1, regex2);
  update i2b2metadata.i2b2_secure set c_dimcode=REGEXP_REPLACE(c_dimcode, regex1, regex2);
  update i2b2metadata.i2b2_secure set c_tooltip=REGEXP_REPLACE(c_tooltip, regex1, regex2);
  update i2b2metadata.i2b2_secure set c_name=newTitle where c_fullname=newTopNode;
 
  update I2B2DEMODATA.concept_counts set concept_path=REGEXP_REPLACE(concept_path, regex1, regex2);
  update I2B2DEMODATA.concept_counts set parent_concept_path=REGEXP_REPLACE(parent_concept_path, regex1, regex2);
 
  update I2B2DEMODATA.concept_dimension set concept_path=REGEXP_REPLACE(concept_path, regex1, regex2);
  update I2B2DEMODATA.concept_dimension set name_char=newTitle where concept_path=newTopNode;
  
  commit;
 
END;
 
$$;


ALTER FUNCTION tm_cz.rename_study(programname character varying, oldtitle character varying, newtitle character varying) OWNER TO tm_cz;

--
-- Name: rwg_add_search_term(text, text, text, bigint); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.rwg_add_search_term(new_term text, category_name text, category_display text, currentjobid bigint DEFAULT NULL::bigint) RETURNS bigint
    LANGUAGE plpgsql
    AS $$
DECLARE

/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
	--Audit variables
	newJobFlag    smallint;
	databaseName  varchar(100);
	procedureName varchar(100);
	jobID         bigint;
	stepCt        bigint;
	rowCt         bigint;
	errorNumber   varchar;
	errorMessage  varchar;

	Parent_Id          integer;
	new_Term_Id        integer;
	keyword_id         integer;
	Lcount             integer;
	Ncount             integer;
	v_category_display varchar(200);


BEGIN
	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;
	SELECT current_user INTO databaseName; --(sic)
	procedureName := 'RWG_ADD_SEARCH_TERM';

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it
	IF (coalesce(jobID::text, '') = '' OR jobID < 1)
		THEN
		newJobFlag := 1; -- True
		SELECT cz_start_audit(procedureName, databaseName) INTO jobID;
	END IF;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Start FUNCTION', 0, stepCt, 'Done');
	stepCt := 1;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Term to add is ''' || new_term || ''', category is ''' ||
		category_name || '''', 0, stepCt, 'Done');

	/*
	1. Check if term exists in Search_Keyword_term
	2. Insert term into Searchapp.search_keyword
	3. Insert term into Searchapp.Search_Keyword_term
	 */
	/*
	-- check if the new term exists
	Select Count(*) 
	into Ncount
	From Searchapp.Search_Keyword
	where upper(Keyword) like upper(New_Term)
	and upper(Display_Data_Category) like upper(category_name);
	If(Ncount>0) Then
		RAISE Existing_Term;
	END IF;
    */

	IF coalesce(category_display::text, '') = '' THEN
		v_category_display := category_name;
	ELSE
		v_category_display := category_display;
	END IF;

	-- Insert taxonomy term into searchapp.search_keyword
	-- (searches Search_Keyword with the parent term to find the category to use)
	BEGIN
	INSERT INTO Searchapp.Search_Keyword(
		Data_Category,
		Keyword,
		Unique_Id,
		Source_Code,
		Display_Data_Category)
	SELECT
		category_name,
		New_Term,
		'RWG:'|| category_name || ':' || New_Term,
		'RWG_ADD_SEARCH_TERM',
		v_category_display
	WHERE
		NOT EXISTS
			(SELECT 1 FROM searchapp.search_keyword x
			WHERE upper(x.data_category) = upper(category_name)
				AND upper(x.keyword) = upper(New_Term));

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Term added to Searchapp.Search_Keyword', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

	-- Get the ID of the new term in Search_Keyword
	BEGIN
	SELECT Search_Keyword_Id INTO Keyword_Id 
	FROM Searchapp.Search_Keyword
	WHERE Upper(Keyword) = Upper(New_Term)
		AND upper(data_category) = upper(category_name);

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'New search keyword ID stored in Keyword_Id', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

	-- Insert the new term into Searchapp.Search_Keyword_Term 
	BEGIN
	INSERT INTO Searchapp.Search_Keyword_Term(
		Keyword_Term,
		Search_Keyword_Id,
		Rank,
		Term_Length)
	SELECT
		New_Term,
		Keyword_Id,
		1,
		Length(New_Term)
	WHERE
		NOT EXISTS (
			SELECT
				1
			FROM
				searchapp.search_keyword_term x
			WHERE
				UPPER(x.keyword_term) = UPPER(New_Term)
				AND x.search_keyword_id = Keyword_Id);

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Term added to Searchapp.Search_Keyword_Term', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

     ---Cleanup OVERALL JOB if this proc is being run standalone    
	IF newJobFlag = 1
		THEN
		PERFORM cz_end_audit (jobID, 'SUCCESS');
	END IF;

	RETURN 0;
EXCEPTION
	WHEN OTHERS THEN
	errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
END;

$$;


ALTER FUNCTION tm_cz.rwg_add_search_term(new_term text, category_name text, category_display text, currentjobid bigint) OWNER TO tm_cz;

--
-- Name: rwg_add_taxonomy_term(text, text, text, bigint); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.rwg_add_taxonomy_term(new_term_in text, parent_term_in text, category_term_in text, currentjobid bigint DEFAULT NULL::bigint) RETURNS bigint
    LANGUAGE plpgsql
    AS $$
DECLARE

/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
--Audit variables
	newJobFlag    smallint;
	databaseName  varchar(100);
	procedureName varchar(100);
	jobID         bigint;
	stepCt        bigint;
	rowCt         bigint;
	errorNumber   varchar;
	errorMessage  varchar;


	Parent_Id integer;
	New_Term_in_Id integer;
	keyword_id integer;
	Lcount integer;
	Ncount integer;
	--Existing_Term Exception;
BEGIN
	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;
	SELECT current_user INTO databaseName; --(sic)
	procedureName := 'RWG_ADD_TAXONOMY_TERM';

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it
	IF (coalesce(jobID::text, '') = '' OR jobID < 1)
		THEN
		newJobFlag := 1; -- True
		SELECT cz_start_audit(procedureName, databaseName) INTO jobID;
	END IF;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Start FUNCTION', 0, stepCt, 'Done');
	stepCt := 1;

	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Term: ' || New_Term_in, 0, stepCt, 'Done');
	stepCt := stepCt + 1;

	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Category: ' || category_term_in, 0, stepCt, 'Done');
	stepCt := stepCt + 1;

	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Parent: ' || parent_term_in, 0, stepCt, 'Done');
	stepCt := stepCt + 1;

	/*
	0. Check if term exists in Search_Keyword_term
	1. Insert term into Searchapp.search_keyword
	2. Insert term into Searchapp.Search_Keyword_term
	3. Find parent
	4. Insert new term into Searchapp.Search_Taxonomy
	5. Find id of new term
	6. Insert relationship into searchapp.search_taxonomy_rels
	 */
	-- Get the data category using the parent term
	/*
	Select distinct(data_category)
	into category_term_in
	From Searchapp.Search_Keyword Where Upper(Keyword)
	like upper(parent_term_in) or upper(display_data_category) like upper(parent_term_in);
	 */
	-- check if the new term exists (use the keyword AND the category, as the same
	-- term name may be used in more than 1 category
	SELECT
		COUNT ( * )
		INTO Ncount
	FROM
		Searchapp.Search_Keyword
	WHERE
		UPPER ( Keyword ) = UPPER ( New_Term_in )
		AND UPPER ( data_category ) LIKE UPPER ( category_term_in );
	--If(Ncount>0) Then
	-- RAISE Existing_Term;
	--END IF;
	-- Insert taxonomy term into searchapp.search_keyword
	IF Ncount = 0 THEN
		BEGIN
		INSERT INTO Searchapp.Search_Keyword (
			Data_Category,
			Keyword,
			Unique_Id,
			Source_Code,
			Display_Data_Category )
		SELECT
			DISTINCT data_category,
			New_Term_in,
			'RWG:' || data_category || ':' || New_Term_in,
			'RWG_ADD_TAXONOMY_TERM',
			Display_Data_Category
		FROM
			Searchapp.Search_Keyword
		WHERE
			UPPER ( display_data_category ) LIKE UPPER ( category_term_in );
		/*
		Insert Into Searchapp.Search_Keyword (Data_Category, Keyword, Unique_Id, Source_Code, Display_Data_Category)
		Select category_term_in, New_Term_in, 'RWG:'|| category_term_in || ':' || New_Term_in, 'RWG_ADD_TAXONOMY_TERM', category_term_in from dual;
		 */
		GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Term added to Searchapp.Search_Keyword', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;
	END IF;

	-- Get the ID of the new term in Search_Keyword
	BEGIN
	SELECT
		Search_Keyword_Id INTO Keyword_Id
	FROM
		Searchapp.Search_Keyword
	WHERE
		UPPER ( Keyword ) = UPPER ( New_Term_in )
		AND UPPER ( data_category ) LIKE UPPER ( category_term_in );
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'New search keyword ID stored in Keyword_Id', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

	-- Insert the new term into Searchapp.Search_Keyword_Term
	BEGIN
	INSERT INTO Searchapp.Search_Keyword_Term (
		Keyword_Term,
		Search_Keyword_Id,
		RANK,
		Term_Length )
	SELECT
		New_Term_in,
		Keyword_Id,
		1,
		LENGTH ( New_Term_in )
	WHERE
		NOT EXISTS (
			SELECT
				1
			FROM
				searchapp.search_keyword_term x
			WHERE
				x.search_keyword_id = Keyword_Id );
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Term added to Searchapp.Search_Keyword_Term', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

	-- Get the ID of the parent term
	SELECT
		DISTINCT Term_Id
	INTO Parent_Id
	FROM
		Searchapp.Search_Taxonomy
	WHERE
		UPPER ( Term_Name ) LIKE UPPER ( parent_term_in );

	IF COALESCE(Parent_Id,-1) > 0 THEN
		-- Insert the new term into the taxonomy
		BEGIN
		INSERT INTO Searchapp.Search_Taxonomy (
			term_name,
			source_cd,
			import_date,
			search_keyword_id )
		SELECT
			New_Term_in,
			parent_term_in || ':' || New_Term_in,
			LOCALTIMESTAMP,
			Keyword_Id
		WHERE
			NOT EXISTS (
				SELECT
					1
				FROM
					searchapp.search_taxonomy x
				WHERE
					x.search_keyword_id = Keyword_Id );
		GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Term added to Searchapp.Search_Taxonomy', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

		-- Get the ID of the new term
		SELECT
			DISTINCT Term_Id INTO New_Term_in_Id
		FROM
			Searchapp.Search_Taxonomy
		WHERE
			UPPER ( Term_Name ) LIKE UPPER ( New_Term_in );

		BEGIN
		INSERT INTO Searchapp.Search_Taxonomy_Rels (
			Child_Id,
			Parent_Id )
		SELECT
			New_Term_in_Id,
			Parent_Id
		WHERE
			NOT EXISTS (
				SELECT
					1
				FROM
					searchapp.search_taxonomy_rels x
				WHERE
					x.child_id = New_Term_in_Id
					AND x.parent_id = Parent_id );
		GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Term relationship added to Searchapp.Search_Taxonomy_Rels', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;
	END IF;

	---Cleanup OVERALL JOB if this proc is being run standalone
	IF newJobFlag = 1
		THEN
		PERFORM cz_end_audit (jobID, 'SUCCESS');
	END IF;
EXCEPTION
	WHEN OTHERS THEN
	errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
END;
$$;


ALTER FUNCTION tm_cz.rwg_add_taxonomy_term(new_term_in text, parent_term_in text, category_term_in text, currentjobid bigint) OWNER TO tm_cz;

--
-- Name: rwg_create_analysis_entry(text, text, bigint); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.rwg_create_analysis_entry(trialid text, delete_flag text DEFAULT NULL::text, currentjobid bigint DEFAULT NULL::bigint) RETURNS bigint
    LANGUAGE plpgsql
    AS $$
DECLARE
	--Audit variables
	newJobFlag    smallint;
	databaseName  varchar(100);
	procedureName varchar(100);
	jobID         bigint;
	stepCt        bigint;
	rowCt         bigint;
	errorNumber   varchar;
	errorMessage  varchar;

	Cdelete CURSOR FOR 
		SELECT
			DISTINCT ( baa.bio_assay_analysis_id )
		FROM
			Biomart.bio_assay_analysis baa
		WHERE
			UPPER ( baa.etl_id ) LIKE ( UPPER ( trialID ) || ':%' );
	cDeleteRow record;

BEGIN
	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;
	SELECT current_user INTO databaseName; --(sic)
	procedureName := 'RWG_CREATE_ANALYSIS_ENTRY';

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it
	IF (coalesce(jobID::text, '') = '' OR jobID < 1)
		THEN
		newJobFlag := 1; -- True
		SELECT cz_start_audit(procedureName, databaseName) INTO jobID;
	END IF;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Start FUNCTION', 0, stepCt, 'Done');
	stepCt := 1;

	-- If this flag is set to 'D', all study data from biomart.bio_assay_analysis and biomart.bio_assay_analysis_data
	IF upper(delete_flag) = 'D' THEN
		PERFORM cz_write_audit(jobId,databaseName,procedureName,'Start Delete bio_assay_analysis_data Loop',0,stepCt,'Done');
		stepCt := stepCt + 1;	

		FOR Cdeleterow IN Cdelete LOOP
			BEGIN
			DELETE
			FROM
				biomart.bio_assay_analysis_data baad
			WHERE
				baad.bio_assay_analysis_id = cDeleteRow.bio_assay_analysis_id;
			GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Delete records for analysis:  ' || cDeleteRow.bio_assay_analysis_id, rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;
		END LOOP;

		BEGIN
		DELETE
		FROM
			Biomart.bio_assay_analysis baa
		WHERE UPPER ( baa.etl_id ) LIKE ( UPPER ( trialID ) || ':%' );
		GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Delete existing records from Biomart.bio_assay_analysis', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;
	END IF;

	BEGIN
	INSERT INTO Biomart.Bio_Assay_Analysis (
		ANALYSIS_NAME,
		Short_Description,
		Long_Description,
		Fold_Change_Cutoff,
		Pvalue_Cutoff,
		lsmean_cutoff,
		Analysis_Method_Cd,
		Bio_Assay_Data_Type,
		Etl_Id,
		Qa_Criteria,
		analysis_create_date,
		analysis_update_date )
	SELECT
		rwg.analysis_id,
		rwg.Short_Desc,
		rwg.Long_Desc,
		rwg.foldchange_cutoff,
		pvalue_cutoff,
		lsmean_cutoff,
		--fold_change, pvalue, lsmean cutoffs
		rwg.Analysis_Type,
		rwg.Data_Type,
		rwg.study_id || 'RWG',
		'(Abs(fold Change) > ' || rwg.foldchange_cutoff || ' OR coalesce(fold_change::text, '') = '')' || ' AND pvalue < ' || pvalue_cutoff || ' AND Max(LSMean) >' || lsmean_cutoff,
		now(),
		now()
	FROM
		tm_lz.rwg_analysis rwg
	WHERE
		UPPER ( rwg.study_id ) = UPPER ( trialID );
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Insert records into Biomart.Bio_Assay_Analysis', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

	/* Update tm_lz.Rwg_Analysis with the newly created bio_assay_analysis_Id */
	BEGIN
	UPDATE tm_lz.Rwg_Analysis rwg
	SET
		rwg.bio_assay_analysis_id = (
			SELECT
				baa.bio_assay_analysis_id
			FROM
				Biomart.Bio_Assay_Analysis baa
			WHERE
				baa.analysis_name = rwg.analysis_id
				AND UPPER ( baa.etl_id ) LIKE UPPER ( trialID || ':%' )
				AND UPPER ( rwg.study_id ) LIKE UPPER ( trialID ) )
	WHERE
		UPPER ( rwg.study_id ) LIKE UPPER ( trialID );
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Update records in tm_lz.Rwg_Analysis with bio_assay_analysis_id ', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

	PERFORM cz_write_audit(jobId,databaseName,procedureName,'FUNCTION Complete',0,stepCt,'Done');
	RETURN 0;

	---Cleanup OVERALL JOB if this proc is being run standalone
	IF newJobFlag = 1
		THEN
		PERFORM cz_end_audit (jobID, 'SUCCESS');
	END IF;
EXCEPTION
	WHEN OTHERS THEN
	errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
END;
$$;


ALTER FUNCTION tm_cz.rwg_create_analysis_entry(trialid text, delete_flag text, currentjobid bigint) OWNER TO tm_cz;

--
-- Name: rwg_import_from_ext(text, bigint); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.rwg_import_from_ext(trialid text, currentjobid bigint DEFAULT NULL::bigint) RETURNS bigint
    LANGUAGE plpgsql
    AS $$
DECLARE

/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
	newJobFlag    smallint;
	databaseName  varchar(100);
	procedureName varchar(100);
	jobID         bigint;
	stepCt        bigint;
	rowCt         bigint;
	errorNumber   varchar;
	errorMessage  varchar;
BEGIN
	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;
	SELECT current_user INTO databaseName; --(sic)
	procedureName := 'RWG_IMPORT_FROM_EXT';

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it
	IF (coalesce(jobID::text, '') = '' OR jobID < 1)
		THEN
		newJobFlag := 1; -- True
		SELECT cz_start_audit(procedureName, databaseName) INTO jobID;
	END IF;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Start FUNCTION', 0, stepCt, 'Done');
	stepCt := 1;

	delete from TM_LZ.Rwg_Analysis where upper(study_id) =upper(trialID);
	GET DIAGNOSTICS rowCt := ROW_COUNT;

	perform cz_write_audit(jobId,databaseName,procedureName,'Delete existing records from TM_LZ.Rwg_Analysis',rowCt,stepCt,'Done');
	stepCt := stepCt + 1;

	delete from TM_LZ.Rwg_Cohorts where upper(study_id) =upper(trialID);
	GET DIAGNOSTICS rowCt := ROW_COUNT;

	perform cz_Write_Audit(Jobid,Databasename,Procedurename,'Delete existing records from TM_LZ.Rwg_Cohorts',rowCt,Stepct,'Done');
	stepCt := stepCt + 1;

	delete from TM_LZ.Rwg_Samples where upper(study_id) =upper(trialID);
	perform cz_Write_Audit(Jobid,Databasename,Procedurename,'Delete existing records from TM_LZ.Rwg_Samples',rowCt,Stepct,'Done');
	stepCt := stepCt + 1;

	-- not used??
	-- delete from TM_LZ.RWG_BAAD_ID where upper(study_id) =upper(trialID);
	--Cz_Write_Audit(Jobid,Databasename,Procedurename,'Delete existing records from TM_LZ.RWG_BAAD_ID',rowCt,Stepct,'Done');
	--stepCt := stepCt + 1;
	--Insert Analysis
	BEGIN
	INSERT INTO TM_LZ.Rwg_Analysis
	(
		Study_Id,
		Cohorts,
		ANALYSIS_ID,
		pvalue_cutoff,
		foldchange_cutoff ,
		lsmean_cutoff,
		Analysis_Type,
		Data_Type,
		Platform,
		Long_Desc,
		Short_Desc,
		import_date
	)
	SELECT
		Upper(Replace(  Study_Id,'"','')),
		REGEXP_REPLACE(upper(Replace(  Cohorts,'"','')), '\s*', '', 'g'),
		Replace(  ANALYSIS_ID ,'"',''),
		Replace( pvalue_cutoff ,'"','')::double precision,
		Replace(  foldchange_cutoff ,'"','')::double precision,
		Replace( lsmean_cutoff ,'"','')::double precision,
		Replace(  Analysis_Type,'"',''),
		Replace(  Data_Type,'"',''),
		Replace(   Platform,'"',''),
		Replace(  Long_Desc,'"',''),
		Replace(   short_desc,'"',''),
		now()
    FROM  TM_LZ.Rwg_Analysis_Ext
    WHERE upper(study_id) = upper(trialID);
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Insert records from TM_LZ.Rwg_Analysis_Ext to TM_LZ.Rwg_Analysis', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

	--	update bio_assay_analysis_id for any existing analysis_id (20130111 JEA)
	BEGIN
	UPDATE tm_lz.rwg_analysis t
	SET
		bio_assay_analysis_id = (
			SELECT
				b.bio_assay_analysis_id
			FROM
				biomart.bio_assay_analysis b
			WHERE
				b.etl_id = trialID || ':RWG'
				AND b.analysis_name = t.analysis_id )
	WHERE
		t.study_id = trialID
		AND EXISTS (
			SELECT
				1
			FROM
				biomart.bio_assay_analysis x
			WHERE
				x.etl_id = trialID || ':RWG'
				AND t.analysis_id = x.analysis_name );

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Update bio_assay_analysis_id on existing rwg_analysis records', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

    --Insert Cohorts
	BEGIN
	INSERT
	INTO TM_LZ.Rwg_Cohorts
	(
		Study_Id,
		Cohort_Id,
		Cohort_Title, Disease, Long_Desc,
		Organism, Pathology, Sample_Type, Short_Desc, Treatment,IMPORT_DATE
	)
	SELECT
	Upper(Replace(Study_Id,'"','')),
	trim(upper(Replace(Cohort_Id,'"',''))),
	Replace(Cohort_Title, '"',''),
	Replace(Disease, '"',''),
	Replace(Long_Desc, '"',''),
	Replace(Organism, '"',''),
	Replace(Pathology, '"',''),
	Replace(Sample_Type, '"',''),
	Replace(Short_Desc, '"',''),
	Replace(Treatment,'"',''),
	now()
	From  TM_LZ.Rwg_Cohorts_Ext
	where upper(study_id) = upper(trialID);

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Insert records from TM_LZ.Rwg_Cohorts_Ext to TM_LZ.Rwg_Cohorts', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

	--Insert samples
	BEGIN
	INSERT
	INTO TM_LZ.Rwg_Samples(
		study_id, COHORTS, EXPR_ID, IMPORT_DATE
	)
	SELECT
		Upper(Replace(  Study_Id,'"','')),
		trim(upper(Replace(  Cohorts,'"',''))),
		Replace(  Expr_Id, '"',''),
		now()
	FROM  TM_LZ.Rwg_Samples_Ext
	WHERE upper(study_id) = upper(trialID);

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Insert records from TM_LZ.Rwg_Samples_Ext to TM_LZ.Rwg_Samples', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

	---Cleanup OVERALL JOB if this proc is being run standalone
	IF newJobFlag = 1 THEN
		PERFORM cz_end_audit(jobID, 'SUCCESS');
	END IF;

	RETURN 0;
EXCEPTION
	WHEN OTHERS THEN
	errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
END;

$$;


ALTER FUNCTION tm_cz.rwg_import_from_ext(trialid text, currentjobid bigint) OWNER TO tm_cz;

--
-- Name: rwg_load_analysis_data(text, bigint, bigint); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.rwg_load_analysis_data(trialid text, currentjobid bigint DEFAULT NULL::bigint, inplatformid bigint DEFAULT NULL::bigint, OUT rtn_code bigint) RETURNS bigint
    LANGUAGE plpgsql
    AS $_$
DECLARE

/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
--Audit variables
	newJobFlag    smallint;
	databaseName  varchar(100);
	procedureName varchar(100);
	jobID         bigint;
	stepCt        bigint;
	rowCt         bigint;
	errorNumber   varchar;
	errorMessage  varchar;

	vWZcount      integer;
	vLZcount      integer;
	vPlatformID   integer;
	vExpID        integer;

	cDelete       CURSOR FOR
		SELECT distinct bio_assay_analysis_id
			FROM tm_lz.Rwg_Analysis
			-- From BIOMART.bio_analysis_cohort_xref
			WHERE upper(study_id) = upper(trialID);

    cDeleteRow    record; --type of cDelete rows

BEGIN
	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;
	SELECT current_user INTO databaseName; --(sic)
	procedureName := 'RWG_LOAD_ANALYSIS_DATA';

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it
	IF (coalesce(jobID::text, '') = '' OR jobID < 1)
		THEN
		newJobFlag := 1; -- True
		SELECT cz_start_audit(procedureName, databaseName) INTO jobID;
	END IF;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Start FUNCTION', 0, stepCt, 'Done');
	stepCt := 1;

	FOR cDeleteRow IN cDelete
	LOOP
		BEGIN
		DELETE FROM BIOMART.bio_assay_analysis_data
		WHERE bio_assay_analysis_id = cDeleteRow.bio_assay_analysis_id;
		GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Delete records from BIOMART.bio_assay_analysis_data for analysis:  ' || cDeleteRow.bio_assay_analysis_id, rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

		BEGIN
		DELETE FROM tm_lz.rwg_analysis_data
		WHERE bio_assay_analysis_id = cDeleteRow.bio_assay_analysis_id;
		GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Delete records from BIOMART.bio_assay_analysis_data for analysis:  ' || cDeleteRow.bio_assay_analysis_id, rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;
	END LOOP;

	BEGIN
	TRUNCATE TABLE tm_wz.bio_assay_analysis_data_new;
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Truncate tm_wz.bio_assay_analysis_data_new', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	BEGIN
	TRUNCATE TABLE tm_wz.tmp_assay_analysis_metrics;
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Truncate tm_wz.tmp_assay_analysis_metrics', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	-- not used ???
	--delete from tm_lz.RWG_BAAD_ID where upper(study_id) =upper(trialID);
	--Cz_Write_Audit(Jobid,Databasename,Procedurename,'Delete existing records from tm_lz.RWG_BAAD_ID',rowCt,Stepct,'Done');
	--stepCt := stepCt + 1;
	IF (coalesce(inPlatformID::text, '') = '')
		THEN

		BEGIN
		SELECT max(bap.bio_assay_platform_id) INTO vPlatformID
		FROM DEAPP.de_subject_sample_mapping ssm, DEAPP.de_gpl_info gpl, BIOMART.bio_assay_platform bap
		WHERE upper(ssm.gpl_id) = upper(gpl.platform)
			AND upper(ssm.trial_name) = upper(trialID)
			AND ssm.platform = 'MRNA_AFFYMETRIX'
			AND upper(bap.platform_accession) LIKE '%'|| upper(gpl.platform) || '%';
		GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Get bio_assay_platform_ID: ' || vPlatformID, rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;
	ELSE
		vPlatformID := inPlatformID;
	END IF;

	BEGIN
	SELECT exp.bio_experiment_id INTO vExpID
	FROM BIOMART.bio_experiment exp
	WHERE upper(accession) = upper(trialID);
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Get bio_experiment_id: ' || vExpID, rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	BEGIN
	SELECT count(*) INTO vLZcount FROM TM_LZ.RWG_ANALYSIS_DATA_EXT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Count for TM_LZ.RWG_ANALYSIS_DATA_EXT = ' || vLZcount, 0, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	--	count number of data records with non-numeric data in preferred_pvalue or fold_change and log
	BEGIN
	SELECT COUNT (*)
	INTO vLZcount
	FROM tm_lz.rwg_analysis_data_ext
	WHERE
		is_numeric(preferred_pvalue) = 1
		OR is_numeric(fold_change) = 1;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Data records dropped for non-numeric preferred_pvalue or fold_change', vLZcount, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	--	insert data into rwg_analysis_data, skip records with non-numeric data in preferred_pvalue or fold_change
	--	change all other non-numeric data to null
	BEGIN
	INSERT INTO tm_lz.rwg_analysis_data (
		study_id,
		probeset,
		fold_change,
		pvalue,
		raw_pvalue,
		adjusted_pvalue,
		min_lsmean,
		max_lsmean,
		analysis_cd,
		bio_assay_analysis_id )
	SELECT
		rwg.study_id,
		ext.probeset,
		ext.fold_change::double precision,
		ext.preferred_pvalue::double precision,
		CASE
			WHEN is_numeric ( ext.raw_pvalue ) = 1 THEN NULL
			ELSE ext.raw_pvalue
		END::double precision
		,
		CASE
			WHEN is_numeric ( ext.adjusted_pvalue ) = 1 THEN NULL
			ELSE ext.adjusted_pvalue
		END::double precision
		,
		CASE
			WHEN is_numeric ( ext.lsmean_1 ) = 1 OR is_numeric ( ext.lsmean_1 ) = 1 THEN NULL
			WHEN ext.lsmean_1 > ext.lsmean_2 THEN ext.lsmean_2
			ELSE ext.lsmean_1
		END::double precision --min
		,
		CASE
			WHEN is_numeric ( ext.lsmean_1 ) = 1
			OR is_numeric ( ext.lsmean_1 ) = 1 THEN NULL
			WHEN ext.lsmean_1 > ext.lsmean_2 THEN ext.lsmean_1
			ELSE ext.lsmean_2
		END::double precision --max
		,
		ext.analysis_id,
		rwg.bio_assay_analysis_id
	FROM
		TM_LZ.RWG_ANALYSIS_DATA_EXT ext,
		tm_lz.rwg_analysis rwg
	WHERE
		TRIM ( UPPER ( ext.analysis_id ) ) = TRIM ( UPPER ( rwg.analysis_id ) )
		AND UPPER ( rwg.study_id ) = UPPER ( trialID )
		AND is_numeric ( ext.preferred_pvalue ) = 0
		AND is_numeric ( ext.fold_change ) = 0;
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Insert records into rwg_analysis_data', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	BEGIN
	SELECT COUNT(*) INTO vWZcount FROM tm_lz.rwg_analysis_data
	WHERE study_id = upper(trialID);
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Count for tm_lz.rwg_analysis_data = ' || vWZcount, 0, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;


	BEGIN
	INSERT INTO tm_wz.BIO_ASSAY_ANALYSIS_DATA_NEW (
		fold_change_ratio,
		raw_pvalue,
		adjusted_pvalue,
		bio_assay_analysis_id,
		feature_group_name,
		bio_experiment_id,
		bio_assay_platform_id,
		etl_id,
		preferred_pvalue,
		lsmean1,
		lsmean2,
		bio_assay_feature_group_id )
	SELECT
		rad.fold_change,
		rad.raw_pvalue,
		rad.adjusted_pvalue,
		rad.bio_assay_analysis_id,
		rad.probeset,
		vExpID,
		vPlatformID,
		rad.study_id || ':RWG',
		rad.pvalue,
		rad.min_lsmean,
		rad.max_lsmean,
		bafg.bio_assay_feature_group_id
	FROM
		tm_lz.rwg_analysis_data rad
		LEFT JOIN BIOMART.bio_assay_feature_group bafg ON ( rad.probeset = bafg.feature_group_name )
	WHERE
		rad.study_id = UPPER ( trialID ); -- 20121212 JEA
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Insert records into BIO_ASSAY_ANALYSIS_DATA_NEW', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	/*Calculate TEA Values */
	BEGIN
	INSERT INTO tm_wz.tmp_assay_analysis_metrics
	SELECT
		ad.bio_assay_analysis_id,
		COUNT ( * )
		data_ct,
		AVG ( ad.fold_change_ratio )
		fc_mean,
		Stddev ( Ad.Fold_Change_Ratio )
		Fc_Stddev
	FROM
		tm_wz.BIO_ASSAY_ANALYSIS_DATA_NEW ad
		JOIN biomart.bio_assay_analysis a ON ad.bio_assay_analysis_id = a.bio_assay_analysis_id
	WHERE ( ad.fold_change_ratio IS NOT NULL
		AND ad.fold_change_ratio::text <> '' )
		AND a.bio_assay_data_type <> 'RBM'
	GROUP BY
		ad.bio_assay_analysis_id
	ORDER BY
		data_ct;
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Insert records into tm_wz.tmp_assay_analysis_metrics ', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	BEGIN
	UPDATE tm_wz.BIO_ASSAY_ANALYSIS_DATA_NEW AS d
	SET
		tea_normalized_pvalue = TEA_NPV_PRECOMPUTE(d.fold_change_ratio, m.fc_mean, m.fc_stddev )
	FROM
		tm_wz.tmp_assay_analysis_metrics m
	WHERE d.bio_assay_analysis_id = m.bio_assay_analysis_id
		AND ( d.fold_change_ratio IS NOT NULL
			AND d.fold_change_ratio ::text <> '');
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Update TEA records in tm_lz.BIO_ASSAY_ANALYSIS_DATA_NEW', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	/* Final Insert */
	SET search_path = "$user", public, biomart;
	FOR cDeleterow IN cdElete LOOP
		BEGIN
		INSERT INTO biomart.bio_assay_analysis_data(
			FOLD_CHANGE_RATIO, RAW_PVALUE,ADJUSTED_PVALUE,BIO_ASSAY_ANALYSIS_ID,
			FEATURE_GROUP_NAME,BIO_EXPERIMENT_ID,BIO_ASSAY_PLATFORM_ID ,
			ETL_ID,PREFERRED_PVALUE,TEA_NORMALIZED_PVALUE,BIO_ASSAY_FEATURE_GROUP_ID,
			LSMEAN1,LSMEAN2 )
		SELECT
		FOLD_CHANGE_RATIO,RAW_PVALUE,ADJUSTED_PVALUE,BIO_ASSAY_ANALYSIS_ID,
		FEATURE_GROUP_NAME,BIO_EXPERIMENT_ID,BIO_ASSAY_PLATFORM_ID ,
		ETL_ID,PREFERRED_PVALUE,TEA_NORMALIZED_PVALUE,BIO_ASSAY_FEATURE_GROUP_ID,
		LSMEAN1,LSMEAN2
		FROM tm_wz.BIO_ASSAY_ANALYSIS_DATA_NEW
		WHERE bio_assay_analysis_id = cDeleteRow.bio_assay_analysis_id;

		GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Insert records into biomart.bio_assay_analysis_data for analysis:  ' || cDeleteRow.bio_assay_analysis_id, rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;
	END LOOP;

	PERFORM cz_write_audit(jobId,databaseName,procedureName,'FUNCTION Complete',0,stepCt,'Done');

	---Cleanup OVERALL JOB if this proc is being run standalone
	IF newJobFlag = 1
		THEN
		PERFORM cz_end_audit (jobID, 'SUCCESS');
	END IF;
EXCEPTION
	WHEN OTHERS THEN
	errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
END;

$_$;


ALTER FUNCTION tm_cz.rwg_load_analysis_data(trialid text, currentjobid bigint, inplatformid bigint, OUT rtn_code bigint) OWNER TO tm_cz;

--
-- Name: rwg_load_analysis_metadata(text, text, text, bigint); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.rwg_load_analysis_metadata(trialid text, i_study_data_category text DEFAULT 'Study'::text, i_study_category_display text DEFAULT NULL::text, currentjobid bigint DEFAULT NULL::bigint, OUT rtn_code bigint) RETURNS bigint
    LANGUAGE plpgsql
    AS $$
DECLARE

/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
  --Audit variables
	newJobFlag    smallint;
	databaseName  varchar(100);
	procedureName varchar(100);
	jobID         bigint;
	stepCt        bigint;
	rowCt         bigint;
	errorNumber   varchar;
	errorMessage  varchar;

	Dcount        integer;
	lcount        integer;
	analysisCount integer;
	resultCount   integer;
	innerRet      bigint;

BEGIN
	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;
	SELECT current_user INTO databaseName; --(sic)
	procedureName := 'RWG_LOAD_ANALYSIS_METADATA';

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it
	IF (coalesce(jobID::text, '') = '' OR jobID < 1)
		THEN
		newJobFlag := 1; -- True
		SELECT cz_start_audit(procedureName, databaseName) INTO jobID;
	END IF;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Start FUNCTION', 0, stepCt, 'Done');
	stepCt := 1;

	/*
	 * Before starting, ensure that the incoming analysis IDs match to the
	 * biomart.bio_assay_analysis name.
	 * If not, try to match using the short_desc. Update the analysis_name if
	 * this work; otherwise, quit
	 */
    /*
	 * NOTE: Due to a change in the curation/etl procedures, this step should no
	 * longer be needed.  The bio_assay_analysis_id is updated in
	 * TM_LZ.Rwg_Analysis at time of creation. A check is done, and if the IDs
	 * match, then this step is bypassed
	 */
  
  
	-- get the count of the incoming analysis data
	BEGIN
	SELECT count(*)
	INTO analysisCount 
	FROM TM_LZ.Rwg_Analysis
	WHERE study_id =  Upper(trialID);

	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Analysis count from TM_LZ.Rwg_Analysis =(see count)', analysisCount, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	--see how many of the analysees match by using the cohort analysis name
	BEGIN
	SELECT
		COUNT ( * )
		INTO resultCount
	FROM
		TM_LZ.Rwg_Analysis analysis,
		Biomart.Bio_Assay_Analysis Baa
	WHERE
		analysis.bio_assay_analysis_id = baa.bio_assay_analysis_id --bio_assay_analysis_id in 'TM_LZ.Rwg_Analysis analysis' should already exist
		AND UPPER ( analysis.study_id ) = UPPER ( trialID );

	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Join analysis.Cohorts to Baa.Analysis_Name, Analysis count =(see count)', resultCount, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	IF analysisCount != resultCount THEN  
	  RAISE 'Analysis count mismatch' USING ERRCODE = 'AA001';
	END IF;


	BEGIN
	DELETE FROM Biomart.Bio_Analysis_Cohort_Xref WHERE upper(study_id) = upper(trialID);

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Delete existing records from Biomart.Bio_Analysis_Cohort_Xref', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	BEGIN
	DELETE FROM Biomart.Bio_Analysis_Attribute_Lineage Baal
	WHERE Baal.Bio_Analysis_Attribute_Id IN
	(SELECT DISTINCT Baa.Bio_Analysis_Attribute_Id
	FROM Biomart.Bio_Analysis_Attribute baa WHERE upper(study_id) = upper(trialID));

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Delete existing records from Biomart.Bio_Analysis_Attribute_Lineage', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	BEGIN
	DELETE FROM Biomart.Bio_Analysis_Attribute  WHERE upper(study_id) = upper(trialID);

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Delete existing records from Biomart.Bio_Analysis_Attribute', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	/** Delete study from biomart.bio_assay_cohort table **/
	BEGIN
	DELETE FROM Biomart.Bio_Assay_Cohort WHERE upper(study_id) = upper(trialID);

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Delete existing records from Biomart.Bio_Assay_Cohort', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	/** Populate biomart.bio_assay_cohort table **/
	BEGIN
	INSERT INTO Biomart.Bio_Assay_Cohort (
		Study_Id,
		Cohort_Id,
		Disease,
		Sample_Type,
		Treatment,
		Organism,
		Pathology,
		Cohort_Title,
		Short_Desc,
		Long_Desc,
		Import_Date )
	SELECT
		Study_Id,
		Cohort_Id,
		Disease,
		Sample_Type,
		Treatment,
		Organism,
		Pathology,
		Cohort_Title,
		Short_Desc,
		Long_Desc,
		now()
	FROM
		TM_LZ.Rwg_Cohorts
	WHERE
		UPPER ( Study_Id ) = UPPER ( trialID );
	
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Insert into Biomart.Bio_Assay_Cohort', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	SELECT coalesce(Max(Length(Regexp_Replace(analysis.Cohorts,'[^;]','g'))),0)+1
	INTO dcount
	FROM TM_LZ.Rwg_Analysis analysis;

	FOR lcount IN 1 .. dcount
		LOOP

		BEGIN
		PERFORM cz_write_audit(jobId,databaseName,procedureName,'Starting Bio_Analysis_Cohort_Xref LOOP, pass: ',lcount,stepCt,'Done');
		INSERT INTO Biomart.Bio_Analysis_Cohort_Xref(
			Study_Id,
			Analysis_Cd,
			Cohort_Id,
			Bio_Assay_Analysis_Id
		)
		SELECT
			upper(analysis.Study_Id),
			analysis.Cohorts,
			trim(Parse_Nth_Value(analysis.Cohorts,lcount,';')) AS cohort,
			baa.bio_assay_analysis_id
		FROM TM_LZ.Rwg_Analysis analysis, Biomart.Bio_Assay_Analysis Baa
		WHERE analysis.bio_assay_analysis_id= Baa.bio_assay_analysis_id
			AND Upper(Baa.Etl_Id) LIKE '%' || Upper(Trialid) || '%'
			AND Upper(analysis.Study_Id) LIKE '%' || Upper(Trialid) || '%'
			AND Trim(Parse_Nth_Value(analysis.Cohorts,lcount,';')) IS NOT NULL;

		GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Insert COHORTS into  into Biomart.Bio_Analysis_Cohort_Xref', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;
	END LOOP;


	/*************************************/
	/** POPULATE Bio_Analysis_Attribute **/
	/*************************************/
	--	delete study from cz_rwg_invalid_terms 20121220 JEA
	BEGIN
	DELETE FROM cz_rwg_invalid_terms
	WHERE upper(study_id) = upper(trialID);

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Delete existing data from cz_rwg_invalid_terms', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	--	insert study as search_term term, sp will check if already exists
	SELECT rwg_add_search_term(upper(trialID),i_study_data_category,i_study_category_display,jobId) INTO innerRet;
	IF innerRet <> 0
		THEN
		errorNumber := '000000';
		errorMessage := 'RWG_ADD_SEARCH_TERM() failed. Arguments: ' ||
				upper(trialID) || ', ' || i_study_data_category ||
				', ' || i_study_category_display  || ', ' || jobId;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END IF;

	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Insert study as search_term', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;

	BEGIN
	-- sample_type: check for any records that do not have a match in the taxonomy
	INSERT INTO cz_rwg_Invalid_Terms(
		Study_Id,
		Category_Name,
		Term_Name)
	SELECT DISTINCT
		upper(Cohort.Study_Id),
		upper('sample_type'),
		cohort.sample_type
	FROM TM_LZ.Rwg_Cohorts Cohort
	WHERE upper(Cohort.Study_Id)=upper(trialID)
		AND NOT EXISTS 
			(SELECT Upper(Tax.Term_Name) FROM Searchapp.Search_Taxonomy Tax
			 WHERE Upper(Cohort.sample_type) = Upper(Tax.Term_Name));
	
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'sample_type: register invalid terms (with no match in the taxonomy)', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	-- sample_type: insert terms into attribute table
	BEGIN
	INSERT INTO Biomart.Bio_Analysis_Attribute(
		Study_Id,
		Bio_Assay_Analysis_Id,
		Term_Id,
		Source_Cd)
	SELECT DISTINCT
		upper(Cohort.Study_Id),
		Xref.Bio_Assay_Analysis_Id,
		Tax.Term_Id, Upper('sample_type:' || Cohort.sample_type) 
	FROM
		TM_LZ.Rwg_Cohorts Cohort,
		Biomart.Bio_Analysis_Cohort_Xref Xref,
		Searchapp.Search_Taxonomy Tax
	WHERE upper(Cohort.Cohort_Id) = upper(Xref.Cohort_Id)
		AND upper(Xref.Study_Id) = upper(Cohort.Study_Id)
		AND Upper(Cohort.Sample_Type) = Upper(Tax.Term_Name)
		AND Cohort.Study_Id =upper(trialID); 

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Insert SAMPLE_TYPE into Bio_Analysis_Attribute', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	-- disease: check for any records that do not have a match in the taxonomy
	BEGIN
	INSERT INTO Cz_Rwg_Invalid_Terms (
		Study_Id,
		Category_Name,
		Term_Name )
	SELECT DISTINCT
		UPPER ( Cohort.Study_Id ),
		'disease',
		cohort.disease
	FROM
		TM_LZ.Rwg_Cohorts Cohort
	WHERE
		UPPER ( Cohort.Study_Id )
		= UPPER ( trialID )
		AND NOT EXISTS (
			SELECT
				UPPER ( Tax.Term_Name )
			FROM
				Searchapp.Search_Taxonomy Tax
			WHERE
				UPPER ( Cohort.disease ) = UPPER ( Tax.Term_Name ) );

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'disease: register invalid terms (with no match in the taxonomy)', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	-- disease: insert terms into attribute table
	BEGIN
	INSERT INTO Biomart.Bio_Analysis_Attribute (
		Study_Id,
		Bio_Assay_Analysis_Id,
		Term_Id,
		Source_Cd )
	SELECT DISTINCT
		UPPER ( Cohort.Study_Id ) ,
		Xref.Bio_Assay_Analysis_Id,
		Tax.Term_Id,
		UPPER ( 'disease:' || Cohort.disease )
	FROM
		TM_LZ.Rwg_Cohorts Cohort,
		Biomart.Bio_Analysis_Cohort_Xref Xref,
		Searchapp.Search_Taxonomy Tax
	WHERE
		UPPER ( Cohort.Cohort_Id ) = UPPER ( Xref.Cohort_Id )
		AND UPPER ( Xref.Study_Id ) = UPPER ( Cohort.Study_Id )
		AND UPPER ( Cohort.Disease ) = UPPER ( Tax.Term_Name )
		AND UPPER ( cohort.study_id ) = UPPER ( trialID ); 

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Insert disease into Bio_Analysis_Attribute', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	-- pathology: check for any records that do not have a match in the taxonomy
	BEGIN
	INSERT INTO Cz_Rwg_Invalid_Terms (
		Study_Id,
		Category_Name,
		Term_Name )
	SELECT DISTINCT
		UPPER ( Cohort.Study_Id ),
		'pathology',
		cohort.pathology
	FROM
		TM_LZ.Rwg_Cohorts Cohort
	WHERE
		UPPER ( Cohort.Study_Id ) = UPPER ( trialID )
		AND NOT EXISTS (
			SELECT
				UPPER ( Tax.Term_Name )
			FROM
				Searchapp.Search_Taxonomy Tax
			WHERE
				UPPER ( Cohort.pathology ) = UPPER ( Tax.Term_Name ) );

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'pathology: register invalid terms (with no match in the taxonomy)', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	-- pathology: insert terms into attribute table
	BEGIN
	INSERT INTO Biomart.Bio_Analysis_Attribute (
		Study_Id,
		Bio_Assay_Analysis_Id,
		Term_Id,
		Source_Cd )
	SELECT DISTINCT
		UPPER ( Cohort.Study_Id ),
		Xref.Bio_Assay_Analysis_Id,
		Tax.Term_Id,
		UPPER ( 'pathology:' || Cohort.pathology )
	FROM
		TM_LZ.Rwg_Cohorts Cohort,
		Biomart.Bio_Analysis_Cohort_Xref Xref,
		Searchapp.Search_Taxonomy Tax
	WHERE
		UPPER ( Cohort.Cohort_Id ) = UPPER ( Xref.Cohort_Id )
		AND UPPER ( Xref.Study_Id ) = UPPER ( Cohort.Study_Id )
		AND UPPER ( Cohort.Pathology ) = UPPER ( Tax.Term_Name )
		AND UPPER ( cohort.study_id ) = UPPER ( trialID );

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Insert pathology into Bio_Analysis_Attribute', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	-- LOOP FOR TREATMENT
	SELECT coalesce(Max(Length(Regexp_Replace(Cohort.Treatment,'[^;]', 'g'))),0)+1
	INTO Dcount
	FROM TM_LZ.Rwg_Cohorts Cohort
	WHERE upper(Cohort.Study_Id)=upper(trialID);
	FOR lcount IN 1 .. dcount
		LOOP	
		stepCt := stepCt + 1;
		PERFORM cz_write_audit(jobId,databaseName,procedureName,'Starting COHORT TREATMENT LOOP, pass: ',lcount,stepCt,'Done');

		-- treatment: check for any records that do not have a match in the taxonomy
		BEGIN
		INSERT INTO Cz_Rwg_Invalid_Terms (
			Study_Id,
			Category_Name,
			Term_Name )
		SELECT
			DISTINCT UPPER ( Cohort.Study_Id ) ,
			'treatment',
			TRIM ( Parse_Nth_Value ( cohort.treatment, lcount, ';' ) )
		FROM
			TM_LZ.Rwg_Cohorts Cohort
		WHERE
			UPPER ( Cohort.Study_Id ) = UPPER ( trialID )
			AND NOT EXISTS (
				SELECT
					UPPER ( Tax.Term_Name )
				FROM
					Searchapp.Search_Taxonomy Tax
				WHERE
					UPPER ( TRIM ( Parse_Nth_Value ( cohort.treatment, lcount, ';' ) ) )
					= UPPER ( Tax.Term_Name ) )
			AND TRIM ( Parse_Nth_Value ( cohort.treatment, lcount, ';' ) ) IS NOT NULL;

		-- treatment: insert terms into attribute table
		INSERT INTO biomart.bio_analysis_attribute (
			study_id,
			bio_assay_analysis_id,
			term_id,
			source_cd )
		SELECT DISTINCT
			UPPER ( cohort.study_id ) ,
			xref.bio_assay_analysis_id,
			tax.term_id,
			UPPER ( 'treatment:' || TRIM ( parse_nth_value ( cohort.treatment,
						lcount,
						';' ) ) )
		FROM
			tm_lz.rwg_cohorts cohort,
			biomart.bio_analysis_cohort_xref xref,
			searchapp.search_taxonomy tax
		WHERE
			UPPER ( cohort.cohort_id ) = UPPER ( xref.cohort_id )
			AND UPPER ( xref.study_id ) = UPPER ( cohort.study_id )
			AND UPPER ( TRIM ( parse_nth_value ( cohort.treatment, lcount, ';' ) ) )
				= UPPER ( tax.term_name )
			AND UPPER ( cohort.study_id ) = UPPER ( trialid );

		GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Insert treatment into Bio_Analysis_Attribute (LOOP)', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	END LOOP;

	stepCt := stepCt + 1;
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(Jobid,Databasename,Procedurename,'END TREATMENT LOOP',rowCt,stepCt,'Done');
	-- END TREATMENT LOOP

	-- organism: check for any records that do not have a match in the taxonomy
	BEGIN
	INSERT INTO Cz_Rwg_Invalid_Terms (
		Study_Id,
		Category_Name,
		Term_Name )
	SELECT DISTINCT
		UPPER ( Cohort.Study_Id ) ,
		'organism',
		Cohort.organism
	FROM
		TM_LZ.Rwg_Cohorts Cohort
	WHERE
		UPPER ( Cohort.Study_Id ) = UPPER ( trialID )
		AND NOT EXISTS (
			SELECT
				UPPER ( Tax.Term_Name )
			FROM
				Searchapp.Search_Taxonomy Tax
			WHERE
				UPPER ( Cohort.organism ) = UPPER ( Tax.Term_Name ) );

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'organism: register invalid terms (with no match in the taxonomy)', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	-- organism: insert terms into attribute table
	BEGIN
	INSERT INTO Biomart.Bio_Analysis_Attribute (
		Study_Id,
		Bio_Assay_Analysis_Id,
		Term_Id,
		Source_Cd )
	SELECT DISTINCT
		UPPER ( Cohort.Study_Id ),
		Xref.Bio_Assay_Analysis_Id,
		Tax.Term_Id,
		UPPER ( 'organism:' || Cohort.Organism )
	FROM
		TM_LZ.Rwg_Cohorts Cohort,
		Biomart.Bio_Analysis_Cohort_Xref Xref,
		Searchapp.Search_Taxonomy Tax
	WHERE
		UPPER ( Cohort.Cohort_Id ) = UPPER ( Xref.Cohort_Id )
		AND UPPER ( Xref.Study_Id ) = UPPER ( Cohort.Study_Id )
		AND UPPER ( Cohort.Organism ) = UPPER ( Tax.Term_Name )
		AND UPPER ( cohort.study_id ) = UPPER ( trialID );

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Insert organism into Bio_Analysis_Attribute', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	-- data_type: check for any records that do not have a match in the taxonomy
	BEGIN
	INSERT INTO Cz_Rwg_Invalid_Terms (
		Study_Id,
		Category_Name,
		Term_Name )
	SELECT DISTINCT
		UPPER ( analysis.Study_Id ),
		'data_type',
		analysis.data_type
	FROM
		TM_LZ.Rwg_Analysis analysis
	WHERE
		UPPER ( analysis.Study_Id ) = UPPER ( trialID )
		AND NOT EXISTS (
			SELECT
				UPPER ( Tax.Term_Name )
			FROM
				Searchapp.Search_Taxonomy Tax
			WHERE
				UPPER ( analysis.data_type ) = UPPER ( Tax.Term_Name ) );

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'data_type: register invalid terms (with no match in the taxonomy)', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	-- data_type: insert terms into attribute table
	BEGIN
	INSERT INTO Biomart.Bio_Analysis_Attribute (
		Study_Id,
		Bio_Assay_Analysis_Id,
		Term_Id,
		Source_Cd )
	SELECT
		UPPER ( analysis.Study_Id ),
		Baa.Bio_Assay_Analysis_Id,
		Tax.Term_Id,
		UPPER ( 'data_type:' || analysis.Data_Type )
	FROM
		TM_LZ.Rwg_Analysis analysis,
		Searchapp.Search_Taxonomy Tax,
		Biomart.Bio_Assay_Analysis Baa
	WHERE
		UPPER ( analysis.Data_Type ) = UPPER ( Tax.Term_Name )
		AND analysis.bio_assay_analysis_id = Baa.bio_assay_analysis_id
		AND UPPER ( Baa.Etl_Id )
			LIKE '%' || UPPER ( analysis.Study_Id ) || '%'
		AND UPPER ( analysis.Study_Id ) = UPPER ( trialID );

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Insert data_type into Bio_Analysis_Attribute', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	-- platform: check for any records that do not have a match in the taxonomy
	BEGIN
	INSERT INTO Cz_Rwg_Invalid_Terms (
		study_id,
		category_name,
		term_name )
	SELECT DISTINCT
		analysis.Study_Id,
		'platform',
		analysis.platform
	FROM
		TM_LZ.Rwg_Analysis analysis
	WHERE
		UPPER ( analysis.Study_Id ) = UPPER ( trialID )
		AND NOT EXISTS (
			SELECT
				UPPER ( Tax.Term_Name )
			FROM
				Searchapp.Search_Taxonomy Tax
			WHERE
				UPPER ( analysis.platform ) = UPPER ( Tax.Term_Name ) );

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'platform: register invalid terms (with no match in the taxonomy)', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	-- platform: insert terms into attribute table
	BEGIN
	INSERT INTO Biomart.Bio_Analysis_Attribute (
		Study_Id,
		Bio_Assay_Analysis_Id,
		Term_Id,
		Source_Cd )
	SELECT
		UPPER ( analysis.Study_Id ),
		Baa.Bio_Assay_Analysis_Id,
		Tax.Term_Id,
		UPPER ( 'platform:' || analysis.Platform )
	FROM
		TM_LZ.Rwg_Analysis analysis,
		Searchapp.Search_Taxonomy Tax,
		Biomart.Bio_Assay_Analysis Baa
	WHERE
		UPPER ( analysis.Platform ) = UPPER ( Tax.Term_Name )
		AND analysis.bio_assay_analysis_id = Baa.bio_assay_analysis_id
		AND UPPER ( Baa.Etl_Id )
			LIKE '%' || UPPER ( analysis.Study_Id ) || '%'
		AND UPPER ( analysis.Study_Id ) = UPPER ( trialID );

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Insert platform into Bio_Analysis_Attribute', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	-- LOOP FOR ANALYSIS TYPE
	SELECT coalesce(Max(Length(Regexp_Replace(analysis.Analysis_Type,'[^;]', 'g'))),0)+1
	INTO Dcount
	FROM TM_LZ.Rwg_Analysis analysis
	WHERE upper(analysis.Study_Id)=upper(trialID);

	FOR Lcount IN 1 .. Dcount
		LOOP	 
		stepCt := stepCt + 1;
		PERFORM cz_write_audit(jobId,databaseName,procedureName,'Starting ANALYSIS_TYPE LOOP, pass: ',lcount,stepCt,'Done');

		-- Analysis_Type: check for any records that do not have a match in the taxonomy
		BEGIN
		INSERT INTO Cz_Rwg_Invalid_Terms (
			Study_Id,
			Category_Name,
			Term_Name )
		SELECT DISTINCT
			UPPER ( analysis.Study_Id ),
			'Analysis_Type',
			TRIM ( Parse_Nth_Value ( analysis.Analysis_Type, lcount, ';' ) )
		FROM
			TM_LZ.Rwg_Analysis analysis
		WHERE
			UPPER ( analysis.Study_Id ) = UPPER ( trialID )
			AND NOT EXISTS (
				SELECT
					UPPER ( Tax.Term_Name )
				FROM
					Searchapp.Search_Taxonomy Tax
				WHERE
					UPPER ( TRIM ( Parse_Nth_Value ( analysis.Analysis_Type, Lcount, ';' ) ) )
						= UPPER ( Tax.Term_Name ) )
			AND TRIM ( Parse_Nth_Value ( analysis.Analysis_Type, Lcount, ';' ) ) IS NOT NULL;

		-- Analysis_Type: insert terms into attribute table
		INSERT INTO Biomart.Bio_Analysis_Attribute (
			Study_Id,
			Bio_Assay_Analysis_Id,
			Term_Id,
			Source_Cd )
		SELECT
			UPPER ( analysis.study_id ),
			baa.bio_assay_analysis_id,
			tax.term_id,
			UPPER ( 'ANALYSIS_TYPE:' || TRIM ( Parse_Nth_Value ( analysis.Analysis_Type,
						lcount,
						';' ) ) )
		FROM
			TM_LZ.Rwg_Analysis analysis,
			Searchapp.Search_Taxonomy Tax,
			Biomart.Bio_Assay_Analysis Baa
		WHERE
			UPPER ( TRIM ( Parse_Nth_Value ( analysis.Analysis_Type,
						Lcount,
						';' ) ) )
				= UPPER ( Tax.Term_Name )
			AND analysis.bio_assay_analysis_id = Baa.bio_assay_analysis_id
			AND UPPER ( Baa.Etl_Id )
				LIKE '%' || UPPER ( analysis.Study_Id ) || '%'
			AND UPPER ( analysis.Study_Id ) = UPPER ( Trialid )
			AND TRIM ( Parse_Nth_Value ( analysis.Analysis_Type, Lcount, ';' ) ) IS NOT NULL;

		GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'LOOP: Insert ANALYSIS_TYPE into Bio_Analysis_Attribute', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;
	END LOOP;

	stepCt := stepCt + 1;
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(Jobid,Databasename,Procedurename,'END ANALYSIS_TYPE LOOP',rowCt,stepCt,'Done');
	-- END ANALYSIS TYPE

	-- search_area: check for any records that do not have a match in the taxonomy
	BEGIN
	INSERT INTO Cz_Rwg_Invalid_Terms (
		Study_Id,
		Category_Name,
		Term_Name )
	SELECT DISTINCT
		UPPER ( ext.Study_Id ),
		'search_area',
		ext.search_area
	FROM
		TM_LZ.clinical_trial_metadata_ext Ext
	WHERE
		UPPER ( ext.Study_Id ) = UPPER ( trialID )
		AND NOT EXISTS (
			SELECT
				UPPER ( Tax.Term_Name )
			FROM
				Searchapp.Search_Taxonomy Tax
			WHERE
				UPPER ( ext.search_area ) = UPPER ( Tax.Term_Name ) );

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'search_area: register invalid terms (with no match in the taxonomy)', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	-- search_area: insert terms into attribute table
	BEGIN
	INSERT INTO Biomart.Bio_Analysis_Attribute (
		Study_Id,
		Bio_Assay_Analysis_Id,
		Term_Id,
		Source_Cd )
	SELECT DISTINCT
		UPPER ( Ext.Study_Id ),
		Xref.Bio_Assay_Analysis_Id,
		Tax.Term_Id,
		UPPER ( 'search_area:' || Ext.search_area )
	FROM
		TM_LZ.Clinical_Trial_Metadata_Ext Ext,
		Biomart.Bio_Analysis_Cohort_Xref Xref,
		Searchapp.Search_Taxonomy Tax
	WHERE
		UPPER ( Xref.Study_Id ) = UPPER ( ext.Study_Id )
		AND UPPER ( Ext.Search_Area ) = UPPER ( Tax.Term_Name )
		AND UPPER ( ext.study_id ) = UPPER ( trialID );

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Insert search_area into Bio_Analysis_Attribute', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	-- data source: check for any records that do not have a match in the taxonomy
	BEGIN
	INSERT INTO Cz_Rwg_Invalid_Terms (
		Study_Id,
		Category_Name,
		Term_Name )
	SELECT DISTINCT
		UPPER ( ext.Study_Id ),
		'DATA_SOURCE',
		ext.data_source
	FROM
		TM_LZ.Clinical_Trial_Metadata_Ext Ext
	WHERE
		UPPER ( ext.Study_Id ) = UPPER ( trialID )
		AND NOT EXISTS (
			SELECT
				UPPER ( Tax.Term_Name )
			FROM
				Searchapp.Search_Taxonomy Tax
			WHERE
				UPPER ( ext.data_source ) = UPPER ( Tax.Term_Name ) );
	
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'data source: register invalid terms (with no match in the taxonomy)', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;
	
	-- data source: insert terms into attribute table
	BEGIN
	INSERT INTO Biomart.Bio_Analysis_Attribute (
		Study_Id,
		Bio_Assay_Analysis_Id,
		Term_Id,
		Source_Cd )
	SELECT DISTINCT
		UPPER ( Ext.Study_Id ),
		Xref.Bio_Assay_Analysis_Id,
		Tax.Term_Id,
		UPPER ( 'DATA_SOURCE:' || Ext.data_source )
	FROM
		TM_LZ.Clinical_Trial_Metadata_Ext Ext,
		Biomart.Bio_Analysis_Cohort_Xref Xref,
		Searchapp.Search_Taxonomy Tax
	WHERE
		UPPER ( Xref.Study_Id ) = UPPER ( ext.Study_Id )
		AND UPPER ( Ext.data_source ) = UPPER ( Tax.Term_Name )
		AND UPPER ( ext.study_id ) = UPPER ( trialID );

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Insert DATA_SOURCE into Bio_Analysis_Attribute', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	-- study_design: check for any records that do not have a match in the taxonomy
	BEGIN
	INSERT INTO Cz_Rwg_Invalid_Terms (
		Study_Id,
		Category_Name,
		Term_Name )
	SELECT DISTINCT
		UPPER ( ext.Study_Id ),
		'study_design',
		ext.study_design
	FROM
		TM_LZ.Clinical_Trial_Metadata_Ext Ext
	WHERE
		UPPER ( ext.Study_Id ) = UPPER ( trialID )
		AND NOT EXISTS (
			SELECT
				UPPER ( Tax.Term_Name )
			FROM
				Searchapp.Search_Taxonomy Tax
			WHERE
				UPPER ( ext.experimental_design ) = UPPER ( Tax.Term_Name ) );
	
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'study design: register invalid terms (with no match in the taxonomy)', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	-- study_design: insert terms into attribute table
	BEGIN
	INSERT INTO Biomart.Bio_Analysis_Attribute (
		Study_Id,
		Bio_Assay_Analysis_Id,
		Term_Id,
		Source_Cd )
	SELECT DISTINCT
		UPPER ( Ext.Study_Id ),
		Xref.Bio_Assay_Analysis_Id,
		Tax.Term_Id,
		UPPER ( 'study_design:' || Ext.experimental_design )
	FROM
		TM_LZ.Clinical_Trial_Metadata_Ext Ext,
		Biomart.Bio_Analysis_Cohort_Xref Xref,
		Searchapp.Search_Taxonomy Tax
	WHERE
		UPPER ( Xref.Study_Id ) = UPPER ( ext.Study_Id )
		AND UPPER ( ( CASE
					WHEN Ext.experimental_design = 'Clinical' THEN 'Clinical Trial'
					ELSE ext.experimental_design
				END ) )
			= UPPER ( Tax.Term_Name )
		AND UPPER ( ext.study_id ) = UPPER ( trialID );

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Insert study_design into Bio_Analysis_Attribute', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;

	--	populate biomart.bio_analysis_attribute_lineage in one shot
	BEGIN
	INSERT INTO biomart.bio_analysis_attribute_lineage (
		bio_analysis_attribute_id,
		ancestor_term_id,
		ancestor_search_keyword_id )
	SELECT
		baa.bio_analysis_attribute_id,
		baa.term_id,
		st.search_keyword_id
	FROM
		biomart.bio_analysis_attribute baa,
		searchapp.search_taxonomy st
	WHERE
		UPPER ( baa.study_id ) = UPPER ( trialID )
		AND baa.term_id = st.term_id;

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Insert attribute links into bio_analysis_attribute_lineage', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;
	/* END populate */

	/*Update the 'analysis_update_date' in bio_assay_analysis (this date is used by solr for incremental updates*/
	BEGIN
	UPDATE BIOMART.bio_assay_analysis baa
	SET
		ANALYSIS_UPDATE_DATE = now()
	WHERE
		UPPER ( baa.etl_id ) LIKE UPPER ( trialID || '%' );

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Update ANALYSIS_UPDATE_DATE with LOCALTIMESTAMP', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
	END;
	PERFORM cz_write_audit(jobId,databaseName,procedureName,'End FUNCTION',0,stepCt,'Done');

	---Cleanup OVERALL JOB if this proc is being run standalone    
	IF newJobFlag = 1
		THEN
		PERFORM cz_end_audit (jobID, 'SUCCESS');
	END IF;
EXCEPTION
	WHEN SQLSTATE 'AA001' then
		stepCt := stepCt + 1;
		GET DIAGNOSTICS rowCt := ROW_COUNT;
		PERFORM cz_write_audit(Jobid,Databasename,Procedurename,'ERR: Check for analysis in rwg_analysis not in biomart.bio_assay_analysis',rowCt,stepCt,'Done');
		--Handle errors.
		PERFORM cz_error_handler (jobID, procedureName, SQLSTATE, SQLERRM);
		--End Proc
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := 16;		
	WHEN OTHERS THEN
	errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		rtn_code := -16;
		RETURN;
END;

$$;


ALTER FUNCTION tm_cz.rwg_load_analysis_metadata(trialid text, i_study_data_category text, i_study_category_display text, currentjobid bigint, OUT rtn_code bigint) OWNER TO tm_cz;

--
-- Name: rwg_load_heat_map_results(text, bigint); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.rwg_load_heat_map_results(in_study_id text, currentjobid bigint DEFAULT NULL::bigint) RETURNS bigint
    LANGUAGE plpgsql
    AS $$
DECLARE

/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/

	--Audit variables
	newJobFlag    smallint;
	databaseName  varchar(100);
	procedureName varchar(100);
	jobID         bigint;
	stepCt        bigint;
	rowCt         bigint;
	errorNumber   varchar;
	errorMessage  varchar;

	sqlText    varchar(500);
	partExists boolean;
	partTable  text;
	i          integer;
	/*
	cursor cInsert is
	Select Distinct Decode(B1.Study_Id, 'C0524T03_RWG', 'C0524T03', B1.Study_Id) Study_Id, B1.Bio_Assay_Analysis_Id, Cohort_Id
	from biomart.bio_analysis_attribute b1, biomart.bio_analysis_cohort_xref b2
	Where B1.Bio_Assay_Analysis_Id = B2.Bio_Assay_Analysis_Id
	and upper(b1.study_id)=upper(in_study_id);
	Cursor Cdelete Is
	Select Distinct Bio_Assay_Analysis_Id
	From biomart.Heat_Map_Results
	where upper(trial_name) = upper(in_study_id);
	cursor ctaIds is
	select bio_assay_analysis_id
	from biomart.bio_assay_analysis
	where etl_id = upper(in_study_id) || ':RWG';
	cInsertRow cInsert%rowtype;
	cDeleteRow cDelete%rowtype;
	cCtaId ctaIds%rowtype;
	 */

BEGIN
	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;
	SELECT current_user INTO databaseName; --(sic)
	procedureName := 'RWG_LOAD_HEAT_MAP_RESULTS';

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it
	IF (coalesce(jobID::text, '') = '' OR jobID < 1)
		THEN
		newJobFlag := 1; -- True
		SELECT cz_start_audit(procedureName, databaseName) INTO jobID;
	END IF;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Start FUNCTION', 0, stepCt, 'Done');
	stepCt := 1;

	partTable := 'heat_map_results_' || lower(in_study_id);

	--	check if partition exists
	SELECT
		EXISTS (
			SELECT
				*
			FROM
				pg_tables
			WHERE
				schemaname = 'biomart'
				AND tablename = partTable )
		INTO partExists;

	IF NOT partExists THEN
		--	needed to add partition to table
		sqlText := 'CREATE TABLE biomart.' || partTable || '(' ||
			'CHECK (trial_name = ''' || upper(in_study_id) || ''') ' ||
			')'
			'INHERITS (biomart.heat_map_results) ' ||
			'TABLESPACE indx';

		BEGIN
		EXECUTE(sqlText);
		PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Adding partition to BIOMART.HEAT_MAP_RESULTS', 0, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

		sqlText := 'CREATE OR REPLACE RULE ' || partTable || '_insert AS ' ||
			'ON INSERT TO biomart.heat_map_results WHERE ' ||
			'(trial_name = ''' || upper(in_study_id) || ''') ' ||
			'DO INSTEAD INSERT INTO biomart.' || partTable || ' ' ||
			'VALUES(NEW.*)';
		BEGIN
		EXECUTE(sqlText);
		PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Adding rule to main table', 0, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

		-- this index is small and very effective for making the UI responsive
		sqlText := 'CREATE INDEX ' || partTable || '_sign_index ' ||
			'ON biomart.' || partTable || '(significant) ' ||
			'TABLESPACE indx';
		BEGIN
		EXECUTE(sqlText);
		PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Adding index to partition', 0, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;
	ELSE
		--truncate partition
		sqlText := 'TRUNCATE TABLE biomart.' || partTable;

		BEGIN
		EXECUTE(sqlText);
		GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Truncate partition in BIOMART.HEAT_MAP_RESULTS', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;
	END IF;

	--	Delete existing data for study from cta_results
	BEGIN
	DELETE
	FROM
		biomart.cta_results
	WHERE
		bio_assay_analysis_id IN (
			SELECT
				x.bio_assay_analysis_id
			FROM
				biomart.bio_assay_analysis x
			WHERE
				x.etl_id = UPPER ( in_study_id ) || ':RWG' );

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Delete records for study from cta_results', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

	--	changed to use sql instead of view, view pulled back all studies   20121203 JEA
	BEGIN
	INSERT INTO biomart.heat_map_results (
		subject_id,
		log_intensity,
		cohort_id,
		probe_id,
		bio_assay_feature_group_id,
		fold_change_ratio,
		tea_normalized_pvalue,
		bio_marker_name,
		bio_marker_id,
		search_keyword_id,
		bio_assay_analysis_id,
		trial_name,
		significant,
		gene_id,
		assay_id,
		preferred_pvalue )
	SELECT
		REPLACE ( REPLACE ( pd.sourcesystem_cd, xref.study_id, '' ),
			':', '' ) AS subject_id,
		md.log_intensity,
		cex.cohort_id,
		dma.probeset,
		baad.bio_assay_feature_group_id,
		baad.Fold_Change_Ratio,
		baad.tea_normalized_pvalue AS tea_normalized_pvalue,
		f.bio_marker_name,
		f.bio_marker_id,
		i.SEARCH_KEYWORD_ID,
		xref.bio_assay_analysis_id,
		xref.study_id,
		CASE
			WHEN ( ABS ( baaD.Fold_Change_Ratio ) > baa.Fold_Change_Cutoff
				OR COALESCE ( baaD.Fold_Change_Ratio ::TEXT, '' ) = '' )
			AND COALESCE ( baad.preferred_pvalue, baad.tea_normalized_pvalue )
				< baa.pvalue_cutoff
			AND ( ( baad.lsmean1 > baa.lsmean_cutoff
					OR baad.lsmean2 > baa.lsmean_cutoff )
				OR ( COALESCE ( baad.lsmean1 ::TEXT, '' ) = ''
					AND COALESCE ( baad.lsmean2 ::TEXT, '' ) = '' ) )
			THEN 1
			ELSE 0
		END Significant,
		f.Primary_External_Id,
		sm.assay_id,
		baad.preferred_pvalue
	FROM
		biomart.bio_analysis_cohort_xref xref
	INNER
		JOIN biomart.bio_cohort_exp_xref cex ON xref.study_id = cex.study_id
		AND xref.cohort_id = cex.cohort_id
	INNER
		JOIN deapp.de_subject_sample_mapping sm ON xref.study_id = sm.trial_name
		AND cex.exp_id = sm.assay_id::varchar
	INNER
		JOIN deapp.de_subject_microarray_data md ON md.trial_name = xref.study_id
		AND cex.exp_id = md.assay_id::varchar
	INNER
		JOIN tm_cz.probeset_deapp dma -- use tm_cz.probeset_deapp because there is only a single record for the probeset
		ON md.probeset_id = dma.probeset_id
	INNER
		JOIN i2b2demodata.patient_dimension pd ON sm.patient_id = pd.patient_num
	INNER
		JOIN biomart.bio_assay_analysis_data baad ON xref.bio_assay_analysis_id = baad.bio_assay_analysis_id
		AND baad.feature_group_name = dma.probeset
	INNER JOIN biomart.bio_assay_data_annotation e ON e.bio_assay_feature_group_id = baad.bio_assay_feature_group_id
	INNER JOIN biomart.bio_marker f ON f.bio_marker_id = e.bio_marker_id
	INNER
		JOIN biomart.bio_assay_analysis baa ON xref.bio_assay_analysis_id = baa.bio_assay_analysis_id
	INNER
		JOIN biomart.bio_marker_correl_mv h ON f.bio_marker_id = h.asso_bio_marker_id
		AND h.correl_type IN ( 'GENE', 'HOMOLOGENE_GENE', 'PROTEIN TO GENE' )
	INNER
		JOIN searchapp.search_keyword i ON f.bio_marker_id = i.bio_data_id
	WHERE
		xref.study_id = UPPER ( in_study_id );

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Insert study to heat_map_results', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

	BEGIN
	UPDATE BIOMART.bio_assay_analysis baa
	SET
		analysis_update_date = now()
	WHERE
		baa.bio_assay_analysis_id IN (
			SELECT
				x.bio_assay_analysis_id
			FROM
				biomart.bio_assay_analysis x
			WHERE
				x.etl_id = UPPER ( in_study_id ) || ':RWG' );

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Updated analysis_update_date for analyses of study', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

	BEGIN
	INSERT INTO biomart.cta_results (
		bio_assay_analysis_id,
		search_keyword_id,
		keyword,
		bio_marker_id,
		bio_marker_name,
		gene_id,
		probe_id,
		fold_change,
		preferred_pvalue,
		organism )
	SELECT
		DISTINCT h.bio_assay_analysis_id,
		h.search_keyword_id,
		UPPER ( s.keyword ),
		h.bio_marker_id,
		b.bio_marker_name,
		b.primary_external_id,
		h.probe_id,
		h.fold_change_ratio,
		h.preferred_pvalue,
		b.organism
	FROM
		biomart.heat_map_results h,
		biomart.bio_marker b,
		searchapp.search_keyword s
	WHERE
		h.trial_name = UPPER ( in_study_id )
		AND h.bio_marker_id = b.bio_marker_id
		AND h.search_keyword_id = s.search_keyword_id;

	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Insert records for study into cta_results', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

	PERFORM cz_write_audit(jobId,databaseName,procedureName,'FUNCTION Complete',0,stepCt,'Done');
	RETURN 0;

	---Cleanup OVERALL JOB if this proc is being run standalone
	IF newJobFlag = 1
		THEN
		PERFORM cz_end_audit (jobID, 'SUCCESS');
	END IF;
EXCEPTION
	WHEN OTHERS THEN
	errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
END;
$$;


ALTER FUNCTION tm_cz.rwg_load_heat_map_results(in_study_id text, currentjobid bigint) OWNER TO tm_cz;

--
-- Name: rwg_remove_study(text, bigint); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.rwg_remove_study(trialid text, currentjobid bigint DEFAULT NULL::bigint) RETURNS bigint
    LANGUAGE plpgsql
    AS $$
DECLARE
--Audit variables
	newJobFlag    smallint;
	databaseName  varchar(100);
	procedureName varchar(100);
	jobID         bigint;
	stepCt        bigint;
	rowCt         bigint;
	errorNumber   varchar;
	errorMessage  varchar;

	sqlText      varchar(500);
	partExists   boolean;
	V_BIO_EXP_ID bigint;
	partTable    text;

BEGIN
	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;
	SELECT current_user INTO databaseName; --(sic)
	procedureName := 'RWG_REMOVE_STUDY';

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it
	IF (coalesce(jobID::text, '') = '' OR jobID < 1)
		THEN
		newJobFlag := 1; -- True
		SELECT cz_start_audit(procedureName, databaseName) INTO jobID;
	END IF;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Start FUNCTION', 0, stepCt, 'Done');
	stepCt := 1;

	partTable := 'heat_map_results_' || lower(trialID);

	BEGIN
	SELECT bio_experiment_id INTO V_BIO_EXP_ID
	FROM biomart.bio_experiment
	WHERE UPPER ( accession ) LIKE UPPER ( trialID );
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Retrieved exp id ' || V_BIO_EXP_ID || ' for trial ' || upper(trialID), rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

	--check that the experiment id exists
	IF NOT FOUND THEN
		RAISE 'Experiment is missing';
	END IF;

	/************* Delete existing records for study ******************/
	BEGIN
	DELETE
	FROM
		biomart.bio_analysis_attribute_lineage baal
	WHERE
		baal.bio_analysis_attribute_id IN (
			SELECT
				baa.bio_analysis_attribute_id
			FROM
				biomart.bio_analysis_attribute baa
			WHERE
				UPPER ( baa.study_id ) = UPPER ( trialID ) );
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Delete existing records from Biomart.bio_analysis_attribute_lineage', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

	BEGIN
	DELETE
	FROM
		biomart.bio_analysis_attribute baa
	WHERE
		UPPER ( baa.study_id ) = UPPER ( trialID );
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Delete existing records from Biomart.bio_analysis_attribute', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

	BEGIN
	DELETE
	FROM
		biomart.bio_analysis_cohort_xref bacx
	WHERE
		UPPER ( bacx.study_id ) = UPPER ( trialID );
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Delete existing records from Biomart.bio_analysis_cohort_xref', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

	BEGIN
	DELETE
	FROM
		biomart.bio_cohort_exp_xref bcex
	WHERE
		UPPER ( bcex.study_id ) = UPPER ( trialID );
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Delete existing records from Biomart.bio_cohort_exp_xref', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

	BEGIN
	DELETE
	FROM
		biomart.bio_assay_cohort bac
	WHERE
		UPPER ( bac.study_id ) = UPPER ( trialID );
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Delete existing records from Biomart.bio_assay_cohort', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

	BEGIN
	DELETE
	FROM
		biomart.bio_assay_analysis_data baad
	WHERE
		baad.bio_experiment_id = V_BIO_EXP_ID;
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Delete existing records from Biomart.bio_assay_analysis_data', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

	BEGIN
	DELETE
	FROM
		biomart.bio_assay_analysis baa
	WHERE
		baa.bio_assay_analysis_id IN (
			SELECT
				DISTINCT hmr.bio_assay_analysis_id
			FROM
				biomart.heat_map_results hmr
			WHERE
				hmr.trial_name = UPPER ( trialID ) );
	GET DIAGNOSTICS rowCt := ROW_COUNT;
	PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Delete existing records from Biomart.bio_assay_analysis', rowCt, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;

	/***********************************/
	/**** Remove Heat_map_resultes ***/
	/***********************************/
	/* Check if partition exist, truncate if so, create it if not */
	SELECT
		EXISTS (
			SELECT *
			FROM pg_tables
			WHERE
				schemaname = 'biomart'
				AND tablename = partTable )
    INTO partExists;

	IF partExists THEN
		sqlText := 'DROP TABLE biomart.' || partTable || ' CASCADE';
		BEGIN
		EXECUTE(sqlText);
		PERFORM cz_write_audit(jobId, databaseName, procedureName,
		'Drop partition table of heat_map_results', 0, stepCt, 'Done');
	stepCt := stepCt + 1;
	EXCEPTION
		WHEN OTHERS THEN
		errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
	END;
	END IF;

	PERFORM cz_write_audit(jobId,databaseName,procedureName,'FUNCTION Complete',0,stepCt,'Done');
	RETURN 0;

	---Cleanup OVERALL JOB if this proc is being run standalone
	IF newJobFlag = 1
		THEN
		PERFORM cz_end_audit (jobID, 'SUCCESS');
	END IF;
EXCEPTION
	WHEN OTHERS THEN
	errorNumber := SQLSTATE;
		errorMessage := SQLERRM;
		PERFORM cz_error_handler(jobID, procedureName, errorNumber, errorMessage);
		PERFORM cz_end_audit (jobID, 'FAIL');
		RETURN -16;
END;
$$;


ALTER FUNCTION tm_cz.rwg_remove_study(trialid text, currentjobid bigint) OWNER TO tm_cz;

--
-- Name: set_bio_data_uid_dis(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.set_bio_data_uid_dis() RETURNS void
    LANGUAGE plpgsql
    AS $$
DECLARE

  --jobRunID CONTROL.SYSTEM_JOB_RUN.JOB_RUN_ID%TYPE;
  --jobStepID CONTROL.SYSTEM_JOB_STEP.JOB_STEP_ID%TYPE;
--CREATE or REPLACE  SYNONYM genego for pictor.genego;

BEGIN

-------------------------------------------------------------------------------
-- Loads data from PICTOR into biomart_LZ
--  emt@20090310
--------------------------------------------------------------------------------
--  jobrunid := control.insert_system_job_run('LoadGeneGOPathways', 'Load All Pathways from GENEGO in PICTOR');

begin

  --jobStepID := control.insert_system_job_step(jobRunID, 'Insert disease pathways into bio_marker for GENEGO disease pathways'
  --, 'Insert disease pathways into bio_marker for GENEGO disease pathways', 22);
  EXECUTE 'delete from biomart.bio_data_uid where unique_id in
                    (select bio_disease_uid(mesh_code) from biomart.bio_disease)';
  EXECUTE 'insert into biomart.bio_data_uid(
                    bio_data_id, unique_id, bio_data_type)
                    PERFORM
                    bio_disease_id, bio_disease_uid(mesh_code), ''BIO_DISEASE''
                    from biomart.bio_disease
                    where not exists
                      (select 1 from biomart.bio_data_uid
                      where bio_disease_uid(bio_disease.mesh_code) = bio_data_uid.unique_id)';

  --control.update_system_job_step_pass(jobStepID, SQL%ROWCOUNT);
  commit;

end;
/*
control.update_system_job_run_pass(jobRunID);
exception
when others then
  control.update_system_job_step_fail(jobStepID, sqlcode, sqlerrm()
  , dbms_utility.format_error_stack, dbms_utility.format_error_backtrace);
  control.update_system_job_run_fail(jobRunID);
  */
end;






 
$$;


ALTER FUNCTION tm_cz.set_bio_data_uid_dis() OWNER TO tm_cz;

--
-- Name: set_bio_data_uid_path(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.set_bio_data_uid_path() RETURNS void
    LANGUAGE plpgsql
    AS $$
DECLARE

 -- jobRunID CONTROL.SYSTEM_JOB_RUN.JOB_RUN_ID%TYPE;
 -- jobStepID CONTROL.SYSTEM_JOB_STEP.JOB_STEP_ID%TYPE;
--CREATE or REPLACE  SYNONYM genego for pictor.genego;

BEGIN

-------------------------------------------------------------------------------
-- Loads data from PICTOR into biomart_LZ
--  emt@20090310
--------------------------------------------------------------------------------
  --jobrunid := control.insert_system_job_run('LoadGeneGOPathways', 'Load All Pathways from GENEGO in PICTOR');



  --jobStepID := control.insert_system_job_step(jobRunID, 'Insert disease pathways into bio_marker for GENEGO disease pathways'
  --, 'Insert disease pathways into bio_marker for GENEGO disease pathways', 22);
  EXECUTE 'delete from biomart.bio_data_uid where unique_id in
                    (select biomarker_pathway_uid(primary_source_code, primary_external_id)
                    from bio_marker)';
  EXECUTE 'delete from biomart.bio_data_uid where unique_id in
                    (select biomarker_gene_uid(primary_external_id)
                    from bio_marker)';
  EXECUTE 'insert into biomart.bio_data_uid(
                    bio_data_id, unique_id, bio_data_type)
                    PERFORM
                    bio_marker_id
                    , biomarker_pathway_uid(primary_source_code, primary_external_id)
                    , ''BIO_MARKER.PATHWAY''
                    from bio_marker
                    where bio_marker_type=''PATHWAY''
                    and not exists
                      (select 1 from biomart.bio_data_uid
                      where biomarker_pathway_uid(bio_marker.primary_source_code, bio_marker.primary_external_id) =
                      bio_data_uid.unique_id)';
  EXECUTE 'insert into biomart.bio_data_uid(
                    bio_data_id, unique_id, bio_data_type)
                    PERFORM
                    bio_marker_id
                    , biomarker_gene_uid(primary_external_id)
                    , ''BIO_MARKER.GENE''
                    from bio_marker
                    where bio_marker_type=''GENE''
                    and not exists
                      (select 1 from biomart.bio_data_uid
                      where biomarker_gene_uid(bio_marker.primary_external_id) = bio_data_uid.unique_id)';


  --control.update_system_job_step_pass(jobStepID, SQL%ROWCOUNT);
  commit;


--control.update_system_job_run_pass(jobRunID);
--exception
--when others then
  --control.update_system_job_step_fail(jobStepID, sqlcode, sqlerrm()
  --, dbms_utility.format_error_stack, dbms_utility.format_error_backtrace);
  --control.update_system_job_run_fail(jobRunID);
end;






 
$$;


ALTER FUNCTION tm_cz.set_bio_data_uid_path() OWNER TO tm_cz;

--
-- Name: sf_xtab(character varying, numeric, character varying, bigint); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.sf_xtab(v_variable character varying, v_protocol numeric, v_subject character varying, v_rowid bigint) RETURNS character varying
    LANGUAGE plpgsql
    AS $$
DECLARE


/******************************************************************************
   NAME:       sf_xtab
   PURPOSE:    This function serves to flatten the SAS to Oracle Conversion via
               the SLM Process. 

   REVISIONS:
   Ver        Date        Author           Description
   ---------  ----------  ---------------  ------------------------------------
   1.0        5/14/2009   George Kuebrich

   NOTES:

   Automatically available Auto Replace Keywords:
      Object Name:     sf_xtab
      Sysdate:         5/14/2009
      Date and Time:   5/14/2009, 8:33:45 AM, and 5/14/2009 8:33:45 AM
      Username:         (set in TOAD Options, Procedure Editor)
      Table Name:       (set in the "New PL/SQL Object" dialog)

******************************************************************************/
sqlstmt varchar(1000);
tmpVar varchar(500);

BEGIN
   
   tmpVar := null;
   EXECUTE 'select distinct value into STRICT  tmpVar
     from sideshow_eav a,
          protocol b,
          variable c
    where a.protocol_id=b.protocol_id
      and a.protocol_id=c.protocol_id
      and a.protocol_id=2 
      and a.variable_id=c.variable_id
      and a.subject_id=3
      and c.variable_name in (4)
      and a.row_id =5' USING v_protocol,v_subject,v_variable,v_rowid;
    
    --EXECUTE IMMEDIATE sqlstmt USING v_protocol,v_subject,v_variable;
  
   RETURN tmpVar;
    EXCEPTION
     WHEN NO_DATA_FOUND THEN
       NULL;
     
END sf_xtab; 
 
$$;


ALTER FUNCTION tm_cz.sf_xtab(v_variable character varying, v_protocol numeric, v_subject character varying, v_rowid bigint) OWNER TO tm_cz;

--
-- Name: tea_npv_precompute(double precision, double precision, double precision); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.tea_npv_precompute(foldchg double precision, mu double precision, sigma double precision) RETURNS double precision
    LANGUAGE plpgsql IMMUTABLE
    AS $$
DECLARE
/*************************************************************************
* Copyright 2008-2012 Janssen Research & Development, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************/
 -------------------------------------------------------------------------------
  -- param foldChg: input is fold change ratio from from analysis_data table
  -- param mu: mean of all analsyis_data records for a given analysis
  -- param sigma: std dev of all analsyis_data records for a given analysis
  -------------------------------------------------------------------------------
  npv double precision;
  outlier_cutoff double precision:=1.0e-5;

BEGIN
  -- normalized p-value
  SELECT 1.0 - tm_cz.cum_normal_dist(abs(foldChg),mu,sigma) INTO npv;

  -- cap outliers to minimum value
  IF npv<outlier_cutoff THEN npv:= outlier_cutoff; END IF;

  RETURN npv;

END;
$$;


ALTER FUNCTION tm_cz.tea_npv_precompute(foldchg double precision, mu double precision, sigma double precision) OWNER TO tm_cz;

--
-- Name: text_parser(text); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.text_parser(text_to_parse text) RETURNS text[]
    LANGUAGE plpgsql
    AS $$
DECLARE

   start_pos bigint;
   end_pos   bigint;
   string_length integer;
   string_tokens varchar(32676);
   counter integer;
   token_value varchar(100);

   list_values text_table;


BEGIN
  -------------------------------------------------------------------------------
   -- Populates a temp_token table with parsed values for any comma separated list.
   -- Requires a type so that multiple records can exist for different uses.
   -- KCR@20090106 - First rev.
   -- Copyright c 2009 Recombinant Data Corp.
   -------------------------------------------------------------------------------

  --Add a delimiter to the end of the string so we dont lose last value
  string_tokens := text_to_parse || ',';

  --Initialize the collection
  list_values := _text() ;

  --get length of string
  string_length := length(string_tokens);

  --set start and end for first token
  start_pos := 1;
  end_pos   := instr(string_tokens,',',1,1);
  counter := 1;
  LOOP
    --Get substring
    token_value := substr(string_tokens, start_pos, end_pos - start_pos);

    --add values to collection
   list_values.EXTEND;
   list_values(list_Values.LAST):= token_value;

    --Check to see if we are done
    IF end_pos = string_length
    THEN
      EXIT;
    ELSE
      -- Increment Start Pos and End Pos
      start_pos := end_pos + 1;
      --increment counter
      counter := counter + 1;
      end_pos := instr(string_tokens, ',',1, counter);

    END IF;
  END LOOP;

  return list_values;

END text_parser;
 
$$;


ALTER FUNCTION tm_cz.text_parser(text_to_parse text) OWNER TO tm_cz;

--
-- Name: tf_trg_antigen_deapp(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.tf_trg_antigen_deapp() RETURNS trigger
    LANGUAGE plpgsql
    AS $$
begin
       if NEW.ANTIGEN_ID is null then
 select nextval('tm_cz.SEQ_ANTIGEN_ID') into NEW.ANTIGEN_ID ;
end if;
       RETURN NEW;
end;
$$;


ALTER FUNCTION tm_cz.tf_trg_antigen_deapp() OWNER TO tm_cz;

--
-- Name: tf_trg_az_test_run_id(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.tf_trg_az_test_run_id() RETURNS trigger
    LANGUAGE plpgsql
    AS $$
begin
       if NEW.TEST_RUN_ID is null then
 select nextval('tm_cz.SEQ_CZ_TEST') into NEW.TEST_RUN_ID ;
end if;
       RETURN NEW;
end;
$$;


ALTER FUNCTION tm_cz.tf_trg_az_test_run_id() OWNER TO tm_cz;

--
-- Name: tf_trg_az_test_step_act_result(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.tf_trg_az_test_step_act_result() RETURNS trigger
    LANGUAGE plpgsql
    AS $$
begin
       if NEW.TEST_STEP_ACT_RESULT_ID is null then
 select nextval('tm_cz.SEQ_CZ_TEST') into NEW.TEST_STEP_ACT_RESULT_ID ;
end if;
       RETURN NEW;
end;
$$;


ALTER FUNCTION tm_cz.tf_trg_az_test_step_act_result() OWNER TO tm_cz;

--
-- Name: tf_trg_az_test_step_run_id(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.tf_trg_az_test_step_run_id() RETURNS trigger
    LANGUAGE plpgsql
    AS $$
begin
       if NEW.TEST_STEP_RUN_ID is null then
 select nextval('tm_cz.SEQ_CZ_TEST') into NEW.TEST_STEP_RUN_ID ;
end if;
       RETURN NEW;
end;
$$;


ALTER FUNCTION tm_cz.tf_trg_az_test_step_run_id() OWNER TO tm_cz;

--
-- Name: tf_trg_cz_data_file_id(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.tf_trg_cz_data_file_id() RETURNS trigger
    LANGUAGE plpgsql
    AS $$
begin
       if NEW.DATA_FILE_ID is null then
 select nextval('tm_cz.SEQ_CZ_DATA_FILE') into NEW.DATA_FILE_ID ;
end if;
       RETURN NEW;
end;
$$;


ALTER FUNCTION tm_cz.tf_trg_cz_data_file_id() OWNER TO tm_cz;

--
-- Name: tf_trg_cz_data_id(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.tf_trg_cz_data_id() RETURNS trigger
    LANGUAGE plpgsql
    AS $$
begin
       if NEW.DATA_ID is null then
 select nextval('tm_cz.SEQ_CZ_DATA') into NEW.DATA_ID ;
end if;
       RETURN NEW;
end;
$$;


ALTER FUNCTION tm_cz.tf_trg_cz_data_id() OWNER TO tm_cz;

--
-- Name: tf_trg_cz_dw_version_id(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.tf_trg_cz_dw_version_id() RETURNS trigger
    LANGUAGE plpgsql
    AS $$
begin
       if NEW.DW_VERSION_ID is null then
 select nextval('tm_cz.SEQ_CZ_DW_VERSION_ID') into NEW.DW_VERSION_ID ;
end if;
       RETURN NEW;
end;
$$;


ALTER FUNCTION tm_cz.tf_trg_cz_dw_version_id() OWNER TO tm_cz;

--
-- Name: tf_trg_cz_job_id(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.tf_trg_cz_job_id() RETURNS trigger
    LANGUAGE plpgsql
    AS $$
begin     
      if NEW.JOB_ID is null then
        select nextval('tm_cz.SEQ_CZ_JOB_MASTER') into NEW.JOB_ID ;       
      end if;       
  RETURN NEW;
end;
$$;


ALTER FUNCTION tm_cz.tf_trg_cz_job_id() OWNER TO tm_cz;

--
-- Name: tf_trg_cz_message_seq_id(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.tf_trg_cz_message_seq_id() RETURNS trigger
    LANGUAGE plpgsql
    AS $$
begin
       if NEW.SEQ_ID is null then
 select nextval('tm_cz.SEQ_CZ_JOB_MESSAGE') into NEW.SEQ_ID ;
end if;
       RETURN NEW;
end;
$$;


ALTER FUNCTION tm_cz.tf_trg_cz_message_seq_id() OWNER TO tm_cz;

--
-- Name: tf_trg_cz_personid(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.tf_trg_cz_personid() RETURNS trigger
    LANGUAGE plpgsql
    AS $$
begin            
if NEW.PERSON_ID is null then          
select nextval('tm_cz.SEQ_CZ_PERSON_ID') into NEW.PERSON_ID ;       end if;      RETURN NEW; end;
$$;


ALTER FUNCTION tm_cz.tf_trg_cz_personid() OWNER TO tm_cz;

--
-- Name: tf_trg_cz_seq_id(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.tf_trg_cz_seq_id() RETURNS trigger
    LANGUAGE plpgsql
    AS $$
begin     
      if NEW.SEQ_ID is null then
        select nextval('tm_cz.SEQ_CZ_JOB_AUDIT') into NEW.SEQ_ID ;       
      end if;       
       RETURN NEW;
  end;
$$;


ALTER FUNCTION tm_cz.tf_trg_cz_seq_id() OWNER TO tm_cz;

--
-- Name: tf_trg_cz_test_category_id(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.tf_trg_cz_test_category_id() RETURNS trigger
    LANGUAGE plpgsql
    AS $$
begin            
if NEW.TEST_CATEGORY_ID is null then          
select nextval('tm_cz.SEQ_CZ') into NEW.TEST_CATEGORY_ID ;       end if;       RETURN NEW;
end;
$$;


ALTER FUNCTION tm_cz.tf_trg_cz_test_category_id() OWNER TO tm_cz;

--
-- Name: tf_trg_cz_test_id(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.tf_trg_cz_test_id() RETURNS trigger
    LANGUAGE plpgsql
    AS $$
begin      
if NEW.TEST_ID is null then          
select nextval('tm_cz.SEQ_CZ') into NEW.TEST_ID ;       end if;       RETURN NEW;
end;
$$;


ALTER FUNCTION tm_cz.tf_trg_cz_test_id() OWNER TO tm_cz;

--
-- Name: tf_trg_cz_test_result_id(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.tf_trg_cz_test_result_id() RETURNS trigger
    LANGUAGE plpgsql
    AS $$
begin     
if NEW.TEST_RESULT_ID is null then          
select nextval('tm_cz.SEQ_CZ') into NEW.TEST_RESULT_ID ;       end if;       RETURN NEW;
end;
$$;


ALTER FUNCTION tm_cz.tf_trg_cz_test_result_id() OWNER TO tm_cz;

--
-- Name: tf_trg_cz_xtrial_ctrl_vocab_id(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.tf_trg_cz_xtrial_ctrl_vocab_id() RETURNS trigger
    LANGUAGE plpgsql
    AS $$
begin     
if NEW.CTRL_VOCAB_ID is null then          
select nextval('tm_cz.SEQ_CZ') into NEW.CTRL_VOCAB_ID ;       end if;       RETURN NEW;
end;
$$;


ALTER FUNCTION tm_cz.tf_trg_cz_xtrial_ctrl_vocab_id() OWNER TO tm_cz;

--
-- Name: tf_trg_probeset_deapp(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.tf_trg_probeset_deapp() RETURNS trigger
    LANGUAGE plpgsql
    AS $$
begin     
			if NEW.PROBESET_ID is null then
				select nextval('tm_cz.SEQ_PROBESET_ID') into NEW.PROBESET_ID ;       
			end if;   
	RETURN NEW;
end;
$$;


ALTER FUNCTION tm_cz.tf_trg_probeset_deapp() OWNER TO tm_cz;

--
-- Name: upgrade_mrna_data(bigint); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.upgrade_mrna_data(currentjobid bigint DEFAULT NULL::bigint) RETURNS void
    LANGUAGE plpgsql
    AS $_$
DECLARE

  
	--Audit variables
	newJobFlag 	integer(1);
	databaseName 	varchar(100);
	procedureName varchar(100);
	jobID 		bigint;
	stepCt 		bigint;
  
	gexStudy	varchar(200);
	gexSource	varchar(200);
	pExists		integer;
	tText		varchar(2000);
  
	Type gex_study_rec is record
	(trial_name	varchar(200)
	)
	);
  
	Type gex_study_tab is table of gex_study_rec;
  
	gex_study_array gex_study_tab;
  
	-- JEA@20120602	New


BEGIN

    stepCt := 0;
	
	--Set Audit Parameters
	newJobFlag := 0; -- False (Default)
	jobID := currentJobID;

	PERFORM sys_context('USERENV', 'CURRENT_SCHEMA') INTO databaseName ;
	procedureName := $$PLSQL_UNIT;

	--Audit JOB Initialization
	--If Job ID does not exist, then this is a single procedure run and we need to create it
	IF(coalesce(jobID::text, '') = '' or jobID < 1)
	THEN
		newJobFlag := 1; -- True
		cz_start_audit (procedureName, databaseName, jobID);
	END IF;
	
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Start upgrade_mrna_data',0,stepCt,'Done');
    COMMIT;
	
	--	get trial_names for all gex data
	
	select distinct trial_name
		  ,coalesce(source_cd,'STD') as source_cd
	bulk collect into gex_study_array
	from de_subject_sample_mapping
	where platform = 'MRNA_AFFYMETRIX'
	order by trial_name
			,source_cd;
	
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Bulk Collect trial_names',SQL%ROWCOUNT,stepCt,'Done');
	
	for i in gex_study_array.first .. gex_study_array.last
	loop
		gexStudy := gex_study_array(i).trial_name;
		gexSource := gex_study_array(i).source_cd;
		
		--	check if new table is partitioned and if partition exists
		
		select count(*) into pExists
		from all_tables
		where table_name = 'DE_SUBJECT_MICROARRAY_DATA_NEW'
		  and partitioned = 'YES';
		  
		if pExists > 0 then
			select count(*) into pExists
			from all_tab_partitions
			where table_name = 'DE_SUBJECT_MICROARRAY_DATA_NEW'
			  and partition_name = gexStudy || ':' || gexSource;
			  
			if pExists = 0 then
				tText := 'alter table deapp.de_subject_microarray_data_new add PARTITION "' || gexStudy || ':' || gexSource || 
						'"  VALUES (' || '''' || gexStudy || ':' || gexSource || '''' || ') ' ||
						   'NOLOGGING COMPRESS TABLESPACE "TRANSMART" ';
				EXECUTE(tText);
				stepCt := stepCt + 1;
				cz_write_audit(jobId,databaseName,procedureName,'Added ' || gexStudy || ':' || gexSource || ' partition to de_subject_microarray_data_new',0,stepCt,'Done');
			end if;
		end if;
		
		insert into deapp.de_subject_microarray_data_new
		(trial_source
		,trial_name
		,probeset_id
		,assay_id
		,patient_id
		,raw_intensity
		,log_intensity
		,zscore
		)
		PERFORM sm.trial_name || ':' || coalesce(sm.source_cd,'STD')
			  ,sm.trial_name
			  ,sd.probeset_id
			  ,sm.assay_id
			  ,sm.patient_id
			  ,sd.raw_intensity
			  ,sd.log_intensity
			  ,sd.zscore
		from de_subject_sample_mapping sm
			,de_subject_microarray_data sd
		where sm.trial_name = gexStudy
		  and sm.source_cd = gexSource
		  and sm.platform = 'MRNA_AFFYMETRIX'
		  and sm.assay_id = sd.assay_id;
		  
		stepCt := stepCt + 1;
		cz_write_audit(jobId,databaseName,procedureName,'Inserted ' || gexStudy || ':' || gexSource || ' to new table',SQL%ROWCOUNT,stepCt,'Done');
		COMMIT;	
			
	end loop;
		
	--	drop indexes on de_subject_microarray_data
	
	i2b2_mrna_index_maint('DROP',null,jobId);
	
	--	rename existing de_subject_microarray_data to _old
	
	EXECUTE('alter table deapp.de_subject_microarray_data rename to de_subject_microarray_data_old');
	
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Rename old de_subject_microarray_data',0,stepCt,'Done');
		
	--	rename _new to de_subject_microarray_data
	
	EXECUTE('alter table deapp.de_subject_microarray_data_new rename to de_subject_microarray_data');
	
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Rename old de_subject_microarray_data',0,stepCt,'Done');
		
	--	add indexes to de_subject_microarray_data
	
	i2b2_mrna_index_maint('ADD',null,jobId);
			
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'End i2b2_audit',0,stepCt,'Done');
	
    COMMIT;
	--Cleanup OVERALL JOB if this proc is being run standalone
	IF newJobFlag = 1
	THEN
		cz_end_audit (jobID, 'SUCCESS');
	END IF;

	EXCEPTION
	WHEN OTHERS THEN
		--Handle errors.
		cz_error_handler (jobID, procedureName);
		--End Proc
		cz_end_audit (jobID, 'FAIL');
	
END;

 
$_$;


ALTER FUNCTION tm_cz.upgrade_mrna_data(currentjobid bigint) OWNER TO tm_cz;

--
-- Name: util_create_synonym_all(character varying, character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.util_create_synonym_all(v_fromzone character varying DEFAULT NULL::character varying, v_whattype character varying DEFAULT 'PROCEDURES,FUNCTIONS,TABLES,VIEWS'::character varying) RETURNS void
    LANGUAGE plpgsql
    AS $$
DECLARE

-------------------------------------------------------------------------------------
-- NAME: UTIL_CREATE or REPLACE _SYNONYM_ALL
--
-- Copyright c 2011 Recombinant Data Corp.
--

--------------------------------------------------------------------------------------
	--The name of the table, proc, function or view.
	V_OBJNAME varchar(50);

	--Dynamic SQL line
	V_CMDLINE varchar(200);

	--Table list
	L_TABLE CURSOR FOR
		SELECT TABLE_NAME FROM ALL_TABLES WHERE OWNER = UPPER(V_FROMZONE);
	--View List
	L_VIEW CURSOR FOR
		SELECT VIEW_NAME FROM ALL_VIEWS WHERE OWNER = UPPER(V_FROMZONE);
	--Procedure and function list (OBJTYPE are PROCEDURE, FUNCTION, TRIGGER)
	L_PROCEDURE CURSOR (OBJTYPE character varying) FOR
		SELECT DISTINCT OBJECT_NAME FROM ALL_PROCEDURES
			WHERE OWNER = UPPER(V_FROMZONE) AND OBJECT_TYPE = OBJTYPE
      AND UPPER(OBJECT_NAME) NOT LIKE 'UTIL%';


BEGIN

	-- Create synonyms for Tables
	IF UPPER(V_WHATTYPE) LIKE '%TABLE%' THEN

		OPEN L_TABLE;
			FETCH L_TABLE INTO V_OBJNAME;
		WHILE L_TABLE%FOUND
			LOOP
			BEGIN

				V_CMDLINE := 'create or replace synonym ' || V_OBJNAME || ' for ' || UPPER(V_FROMZONE) || '.' || V_OBJNAME ;

				EXECUTE V_CMDLINE;
				--DBMS_OUTPUT.PUT_LINE('SUCCESS ' || V_CMDLINE);

				FETCH L_TABLE  INTO V_OBJNAME;

			EXCEPTION
			WHEN OTHERS THEN
			BEGIN
				RAISE NOTICE '%%', 'ERROR ' ,  V_CMDLINE;
				RAISE NOTICE '%', SQLERRM;
			END;
		END;
       END LOOP;
       CLOSE L_TABLE;
   end if;

	--CREATE or REPLACE  SYNONYMS FOR VIEWS
	IF UPPER(V_WHATTYPE) LIKE '%VIEW%' THEN

		OPEN L_VIEW;
			FETCH L_VIEW INTO V_OBJNAME;
		WHILE L_VIEW%FOUND
			LOOP
			BEGIN

				V_CMDLINE := 'create or replace synonym ' || V_OBJNAME || ' for ' || UPPER(V_FROMZONE) || '.' || V_OBJNAME ;

				EXECUTE V_CMDLINE;
				--DBMS_OUTPUT.PUT_LINE('SUCCESS ' || V_CMDLINE);

				FETCH L_VIEW INTO V_OBJNAME;

			EXCEPTION
			WHEN OTHERS THEN
			BEGIN
				RAISE NOTICE '%%', 'ERROR ' ,  V_CMDLINE;
				RAISE NOTICE '%', SQLERRM;
			END;
		END;
		END LOOP;
		CLOSE L_VIEW;
   END IF;

-- CREATE or REPLACE  SYNONYMS FOR PROCEDURES
	IF UPPER(V_WHATTYPE) LIKE '%FUNCTION%' THEN

		OPEN L_PROCEDURE('FUNCTION');
			FETCH L_PROCEDURE INTO V_OBJNAME;
		WHILE L_PROCEDURE%FOUND
			LOOP
			BEGIN

				V_CMDLINE := 'create or replace synonym ' || V_OBJNAME || ' for ' || UPPER(V_FROMZONE) || '.' || V_OBJNAME ;

				EXECUTE V_CMDLINE;
				--DBMS_OUTPUT.PUT_LINE('SUCCESS ' || V_CMDLINE);

				FETCH l_procedure INTO V_OBJNAME;

			EXCEPTION
			WHEN OTHERS THEN
			BEGIN
				RAISE NOTICE '%%', 'ERROR ' ,  V_CMDLINE;
				RAISE NOTICE '%', SQLERRM;
			END;
		END;
		END LOOP;
		CLOSE l_procedure;
   end if;

-- CREATE or REPLACE  SYNONYMS FOR FUNCTIONS
	IF UPPER(V_WHATTYPE) LIKE '%FUNCTION%' THEN

		OPEN l_procedure('FUNCTION');
			FETCH l_procedure INTO V_OBJNAME;
		WHILE l_procedure%FOUND
			LOOP
			BEGIN

				V_CMDLINE := 'create synonym ' || V_OBJNAME || ' for ' || UPPER(V_FROMZONE) || '.' || V_OBJNAME ;

				EXECUTE V_CMDLINE;
				--DBMS_OUTPUT.PUT_LINE('SUCCESS ' || V_CMDLINE);

				FETCH L_PROCEDURE INTO V_OBJNAME;

			EXCEPTION
			WHEN OTHERS THEN
			BEGIN
				RAISE NOTICE '%%', 'ERROR ' ,  V_CMDLINE;
				RAISE NOTICE '%', SQLERRM;
			END;
		END;
		END LOOP;
		CLOSE L_PROCEDURE;
   END IF;
END;
 
$$;


ALTER FUNCTION tm_cz.util_create_synonym_all(v_fromzone character varying, v_whattype character varying) OWNER TO tm_cz;

--
-- Name: util_drop_all(character varying, character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.util_drop_all(v_whattype character varying DEFAULT 'PROCEDURES,FUNCTIONS,VIEWS,SYNONYMS'::character varying, v_dropifempty character varying DEFAULT 'Y'::character varying) RETURNS void
    LANGUAGE plpgsql
    AS $$
DECLARE

-------------------------------------------------------------------------------------
-- NAME: UTIL_DROP_ALL
--
-- Copyright c 2011 Recombinant Data Corp.
--

--------------------------------------------------------------------------------------
   v_procname varchar(50);
   v_objtype varchar(50);
   v_table_name varchar(50);
   v_view_name varchar(50);
   v_synonym_name varchar(50);
   v_constraint_name varchar(50);


   l_table CURSOR FOR
     SELECT table_name from user_tables;

   l_view CURSOR FOR
     SELECT view_name from user_views;

   l_procedure CURSOR FOR
     SELECT distinct object_name, object_type from user_procedures;

   l_synonym CURSOR FOR
     SELECT synonym_name from user_synonyms;

   l_constraint CURSOR FOR
     SELECT distinct table_name, constraint_name from user_constraints;


BEGIN

   --util_make_object_list(v_whattype, v_things);

   -- drop procedure(s) or function(s)
   if upper(v_whattype) like 'FUNCTION' or upper(v_whattype) like 'FUNCTION' then
      open l_procedure;
      fetch l_procedure into v_procname, v_objtype;
      while l_procedure%FOUND
      loop
         -- dbms_output.put_line( v_objtype || '  ' || v_procname);
         EXECUTE 'drop '|| v_objtype || ' ' || v_procname;

         fetch l_procedure into v_procname, v_objtype;
      end loop;
      close l_procedure;
   end if;


   -- drop table(s)
   if upper(v_whattype) like 'TABLE' then
      open l_table;
      fetch l_table into v_table_name;
      while l_table%FOUND
      loop
         -- dbms_output.put_line( v_table_name);
         EXECUTE 'drop table '|| v_table_name || ' cascade constraints ';

         fetch l_table into v_table_name;
      end loop;
      close l_table;
   end if;

   -- drop synonyms(s)
   if upper(v_whattype) like 'SYNONYM' then
      open l_synonym;
      fetch l_synonym into v_synonym_name;
      while l_synonym%FOUND
      loop
         -- dbms_output.put_line( v_synonym_name);
         EXECUTE 'drop synonym ' || v_synonym_name ;

         fetch l_synonym into v_synonym_name;
      end loop;
      close l_synonym;
   end if;


   -- drop view(s)
   if upper(v_whattype) like 'VIEW' then
      open l_view;
      fetch l_view into v_view_name;
      while l_view%FOUND
      loop
         -- dbms_output.put_line( v_view_name);
         EXECUTE 'drop view '|| v_table_name ;

         fetch l_view into v_view_name;
      end loop;
      close l_view;
   end if;


   -- drop constraint(s)
   if upper(v_whattype) like 'CONSTRAINT' then
      open l_constraint;
      fetch l_constraint into v_table_name, v_constraint_name;
      while l_constraint%FOUND
      loop
         -- dbms_output.put_line( v_constraint_name);
         EXECUTE 'alter table '|| v_table_name || ' drop constraint '|| v_constraint_name ;

         fetch l_constraint into v_table_name, v_constraint_name;
      end loop;
      close l_constraint;
   end if;

END;
 
$$;


ALTER FUNCTION tm_cz.util_drop_all(v_whattype character varying, v_dropifempty character varying) OWNER TO tm_cz;

--
-- Name: util_drop_anything(character varying, character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.util_drop_anything(v_objname character varying DEFAULT NULL::character varying, v_objtype character varying DEFAULT NULL::character varying) RETURNS void
    LANGUAGE plpgsql
    AS $$
DECLARE

-------------------------------------------------------------------------------------
-- NAME: UTIL_DROP_ANYTHING
--
-- Copyright c 2011 Recombinant Data Corp.
--

--------------------------------------------------------------------------------------
   v_cmdline varchar(100);


BEGIN

   if upper(v_objtype) like 'TABLE%' then
       v_cmdline := 'drop '|| v_objtype || ' '|| v_objname || ' cascade constraint';
   else
       v_cmdline := 'drop '|| v_objtype || ' '|| v_objname;
   end if;

   BEGIN
      EXECUTE v_cmdline;
      RAISE NOTICE '%%', 'SUCCESS ' ,  v_cmdline;
   EXCEPTION
      WHEN OTHERS THEN
         RAISE NOTICE '%%', 'ERROR ' ,  v_cmdline;
   END;
END;
 
$$;


ALTER FUNCTION tm_cz.util_drop_anything(v_objname character varying, v_objtype character varying) OWNER TO tm_cz;

--
-- Name: util_drop_synonym(character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.util_drop_synonym(v_objname character varying DEFAULT NULL::character varying) RETURNS void
    LANGUAGE plpgsql
    AS $$
DECLARE

-------------------------------------------------------------------------------------
-- NAME: UTIL_DROP_SYNONYM
--
-- Copyright c 2011 Recombinant Data Corp.
--

--------------------------------------------------------------------------------------
   v_cmdline varchar(100);

   ts CURSOR FOR
     SELECT 'drop synonym ' || synonym_name || ' ' from user_synonyms;



BEGIN

  OPEN ts;
   FETCH ts INTO v_cmdline;
   WHILE ts%FOUND
   LOOP

      BEGIN
         BEGIN

            BEGIN
               EXECUTE v_cmdline;
               RAISE NOTICE '%%', 'SUCCESS ' ,  v_cmdline;
            END;
         EXCEPTION
            WHEN OTHERS THEN

               BEGIN
                  RAISE NOTICE '%%', 'ERROR ' ,  v_cmdline;
               END;
         END;
         FETCH ts INTO v_cmdline;
      END;
   END LOOP;
   CLOSE ts;
END;
 
$$;


ALTER FUNCTION tm_cz.util_drop_synonym(v_objname character varying) OWNER TO tm_cz;

--
-- Name: util_drop_synonym_by_owner(character varying, character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.util_drop_synonym_by_owner(v_owner character varying, v_dropifempty character varying DEFAULT 'Y'::character varying) RETURNS void
    LANGUAGE plpgsql
    AS $$
DECLARE

-------------------------------------------------------------------------------------
-- NAME: UTIL_DROP_SYNONYM_BY_OWNER
--
-- Copyright c 2011 Recombinant Data Corp.
--

--------------------------------------------------------------------------------------
   v_procname varchar(50);
   v_objtype varchar(50);
   v_table_name varchar(50);
   v_view_name varchar(50);
   v_synonym_name varchar(50);
   v_constraint_name varchar(50);


   l_synonym CURSOR FOR
     SELECT synonym_name from all_synonyms
	 where owner = v_owner;



BEGIN

   -- drop synonyms(s)

      open l_synonym;
      fetch l_synonym into v_synonym_name;
      while l_synonym%FOUND
      loop
         -- dbms_output.put_line( v_synonym_name);
         EXECUTE( 'drop synonym ' || v_synonym_name) ;

         fetch l_synonym into v_synonym_name;
      end loop;
      close l_synonym;



END;
 
$$;


ALTER FUNCTION tm_cz.util_drop_synonym_by_owner(v_owner character varying, v_dropifempty character varying) OWNER TO tm_cz;

--
-- Name: util_drop_table(character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.util_drop_table(v_tabname character varying DEFAULT NULL::character varying) RETURNS void
    LANGUAGE plpgsql
    AS $$
DECLARE

-------------------------------------------------------------------------------------
-- NAME: UTIL_DROP_TABLE
--
-- Copyright c 2011 Recombinant Data Corp.
--

--------------------------------------------------------------------------------------
  v_exists integer;
  v_cmdline varchar(200);


BEGIN

  --Check if table exists
  select count(*)
  into v_exists
  from user_tables
  where table_name = v_tabname;

  if v_exists > 0 then
     v_cmdline := 'drop table ' || v_tabname;
     EXECUTE v_cmdline;
  end if;

END;
 
$$;


ALTER FUNCTION tm_cz.util_drop_table(v_tabname character varying) OWNER TO tm_cz;

--
-- Name: util_grant_all(character varying, character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.util_grant_all(username character varying DEFAULT 'DATATRUST'::character varying, v_whattype character varying DEFAULT 'PROCEDURES,FUNCTIONS,TABLES,VIEWS,PACKAGES'::character varying) RETURNS void
    LANGUAGE plpgsql
    AS $$
DECLARE

-------------------------------------------------------------------------------------
-- NAME: UTIL_GRANT_ALL
--
-- Copyright c 2011 Recombinant Data Corp.
--

--------------------------------------------------------------------------------------

    --GRANTS DATATRUST POSSIBLE PERMISSIONS
    --ON OBJECTS OWNED BY THE CURRENT USER

	--	JEA@20110901	Added parameter to allow username other than DATATRUST, look for EXTRNL as external table names

    v_user      text2(2000) := SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA');

  
BEGIN

	IF UPPER(V_WHATTYPE) LIKE '%TABLE%' THEN
    RAISE NOTICE '%%%%', 'Owner ' ,  v_user  ,  '   Grantee ' ,  username;
    RAISE NOTICE 'Tables';

     for L_TABLE in (select table_name from user_tables where table_name not like '%EXTRNL%') LOOP

       if L_TABLE.table_name like '%EXTRNL%' then
          --grant select only to External tables
          EXECUTE 'grant select on ' || L_TABLE.table_name || ' to ' || username;

       else
          --Grant full permissions on regular tables
          EXECUTE 'grant select, insert, update, delete on ' || L_TABLE.table_name || ' to ' || username;
          --DBMS_OUTPUT.put_line('grant select, insert, update, delete on ' || L_TABLE.table_name || ' to ' || username);
       end if;

     END LOOP; --TABLE LOOP
     end if;

	IF UPPER(V_WHATTYPE) LIKE '%VIEW%' THEN
    RAISE NOTICE '%%%%', 'Owner ' ,  v_user  ,  '   Grantee ' ,  username;
    RAISE NOTICE 'Views';

     for L_VIEW in (select view_name from user_views ) LOOP
          EXECUTE 'grant select on ' || L_VIEW.view_name || ' to ' || username;

     END LOOP; --TABLE LOOP
 end if;

 IF UPPER(V_WHATTYPE) LIKE '%FUNCTION%' or UPPER(V_WHATTYPE) LIKE '%FUNCTION%' or UPPER(V_WHATTYPE) LIKE '%PACKAGE%' THEN
    RAISE NOTICE '%%', chr(10) ,  'Procedures, functions and packages';

    for L_PROCEDURE in (select object_name from user_objects where object_type in ('FUNCTION', 'FUNCTION', 'PACKAGE') )
     LOOP

       EXECUTE 'grant execute on ' || L_PROCEDURE.object_name || ' to ' || username;
      -- DBMS_OUTPUT.put_line('grant execute on ' || L_PROCEDURE.object_name || ' to ' || username);

     END LOOP; --PROCEDURE LOOP
  end if;

END;
 
$$;


ALTER FUNCTION tm_cz.util_grant_all(username character varying, v_whattype character varying) OWNER TO tm_cz;

--
-- Name: util_grant_execute(character varying, character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.util_grant_execute(v_to_zone character varying DEFAULT NULL::character varying, v_type character varying DEFAULT 'TABLES,VIEWS'::character varying) RETURNS void
    LANGUAGE plpgsql
    AS $$
DECLARE

-------------------------------------------------------------------------------------
-- NAME: UTIL_GRANT_EXECUTE
--
-- Copyright c 2011 Recombinant Data Corp.
--

--------------------------------------------------------------------------------------

   v_object_name varchar(50);
   v_cmdline varchar(200);

   l_table CURSOR FOR
        SELECT table_name from all_tables where owner = v_to_zone;

   l_view CURSOR FOR
        SELECT view_name from all_views where owner = v_to_zone;


BEGIN

   if upper(v_type) like '%TABLE%' then
       OPEN l_table;
       FETCH l_table INTO v_object_name;
       WHILE l_table%FOUND
       LOOP
          BEGIN
             v_cmdline := 'grant select on ' || v_object_name || ' to ' || v_to_zone;

             BEGIN

                BEGIN
                   EXECUTE v_cmdline;
                   RAISE NOTICE '%%', 'SUCCESS ' ,  v_cmdline;
                END;
             EXCEPTION
                WHEN OTHERS THEN

                   BEGIN
                      RAISE NOTICE '%%', 'ERROR ' ,  v_cmdline;
                   END;
             END;

             FETCH l_table INTO v_object_name;
          END;
       END LOOP;
       CLOSE l_table;
   end if;

   if upper(v_type) like '%VIEW%' then
       OPEN l_view;
       FETCH l_view INTO v_object_name;
       WHILE l_view%FOUND
       LOOP
          BEGIN

             v_cmdline := 'grant select on ' || v_object_name || ' to ' || v_to_zone;

             BEGIN

                BEGIN
                   EXECUTE v_cmdline;
                   RAISE NOTICE '%%', 'SUCCESS ' ,  v_cmdline;
                END;
             EXCEPTION
                WHEN OTHERS THEN

                   BEGIN
                      RAISE NOTICE '%%', 'ERROR ' ,  v_cmdline;
                   END;
             END;

             FETCH l_view INTO v_object_name;
          END;
       END LOOP;
       CLOSE l_view;
   end if;

END;
 
$$;


ALTER FUNCTION tm_cz.util_grant_execute(v_to_zone character varying, v_type character varying) OWNER TO tm_cz;

--
-- Name: util_grant_select(character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.util_grant_select(username character varying DEFAULT 'DATATRUST'::character varying) RETURNS void
    LANGUAGE plpgsql
    AS $$
DECLARE

-------------------------------------------------------------------------------------
-- NAME: UTIL_GRANT_ALL
--
-- Copyright c 2011 Recombinant Data Corp.
--

--------------------------------------------------------------------------------------

    --GRANTS SELECT PERMISSIONS to DATATRUST (default) or specified user
    --ON OBJECTS OWNED BY THE CURRENT USER

	--	JEA@20110927	New, cloned from UTIL_GRANT_ALL

    v_user      text2(2000) := SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA');

  
BEGIN

    RAISE NOTICE '%%%%', 'Owner ' ,  v_user  ,  '   Grantee ' ,  username;
    RAISE NOTICE 'Tables';

     for L_TABLE in (select table_name from user_tables) LOOP

	 EXECUTE 'grant select on ' || L_TABLE.table_name || ' to ' || username;
          RAISE NOTICE '%%%%', 'grant select on ' ,  L_TABLE.table_name ,  ' to ' ,  username;

     END LOOP; --TABLE LOOP

 --  dbms_output.put_line(chr(10) || 'Views');

 --   for L_VIEW in (select object_name from user_objects where object_type = 'VIEW' )
  --   LOOP

   --    execute immediate 'grant select on ' || L_VIEW.object_name || ' to ' || username;
   --    DBMS_OUTPUT.put_line('grant select on ' || L_VIEW.object_name || ' to ' || username);

   --  END LOOP; --VIEW LOOP

END;
 
$$;


ALTER FUNCTION tm_cz.util_grant_select(username character varying) OWNER TO tm_cz;

--
-- Name: util_is_populated(character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.util_is_populated(tabname character varying, OUT retval integer) RETURNS integer
    LANGUAGE plpgsql
    AS $$
DECLARE

-------------------------------------------------------------------------------------
-- NAME: UTIL_IS_POPULATED
--
-- Copyright c 2011 Recombinant Data Corp.
--

--------------------------------------------------------------------------------------
  sqltext varchar(500);
  l_count integer;


BEGIN

   sqltext := 'select count(*) into result from ' || tabname;

   EXECUTE sqltext into l_count;


   if l_count > 0 then
   retval :=1;
   else
    retval := 0;
   end if;

   --dbms_output.put_line(l_count);

end;
 
$$;


ALTER FUNCTION tm_cz.util_is_populated(tabname character varying, OUT retval integer) OWNER TO tm_cz;

--
-- Name: util_make_object_list(character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.util_make_object_list(v_whattype character varying DEFAULT NULL::character varying, OUT v_things character varying) RETURNS character varying
    LANGUAGE plpgsql
    AS $$
DECLARE

-------------------------------------------------------------------------------------
-- NAME: UTIL_MAKE_OBJECT_LIST
--
-- Copyright c 2011 Recombinant Data Corp.
--

--------------------------------------------------------------------------------------


BEGIN

   v_things := REPLACE(UPPER(v_whattype), 'PROCEDURES', 'P,PC') ;
   v_things := REPLACE(UPPER(v_things), 'FUNCTION', 'P,PC') ;
   v_things := REPLACE(UPPER(v_things), 'CONSTRAINTS', 'PK,F') ;
   v_things := REPLACE(UPPER(v_things), 'CONSTRAINT', 'PK,F') ;
   v_things := REPLACE(UPPER(v_things), 'FUNCTIONS', 'FN') ;
   v_things := REPLACE(UPPER(v_things), 'FUNCTION', 'FN') ;
   v_things := REPLACE(UPPER(v_things), 'TABLES', 'U') ;
   v_things := REPLACE(UPPER(v_things), 'TABLE', 'U') ;
   v_things := REPLACE(UPPER(v_things), 'VIEWS', 'V') ;
   v_things := REPLACE(UPPER(v_things), 'VIEW', 'V') ;

   -- add more common names for things
   -- but now transform a,b into 'a','b'
   v_things := REPLACE(UPPER(v_things), ',', ''',''') ;
   v_things := '''' || v_things || '''' ;
END;
 
$$;


ALTER FUNCTION tm_cz.util_make_object_list(v_whattype character varying, OUT v_things character varying) OWNER TO tm_cz;

--
-- Name: util_recompile_all(); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.util_recompile_all() RETURNS void
    LANGUAGE plpgsql
    AS $$
DECLARE

-------------------------------------------------------------------------------------
-- NAME: UTIL_RECOMPILE_ALL
--
-- Copyright c 2011 Recombinant Data Corp.
--

--------------------------------------------------------------------------------------
   v_proclist CURSOR FOR
     SELECT distinct 'alter '|| object_type || ' ' || object_name || ' compile '
     FROM user_procedures;

   v_procname varchar(50);


BEGIN

   OPEN v_proclist;
   FETCH v_proclist INTO v_procname;
   WHILE v_proclist%FOUND
   LOOP

      BEGIN
         BEGIN

            BEGIN
               EXECUTE v_procname;
               RAISE NOTICE '%%', 'succesfully compiled ' ,  v_procname;
            END;
         EXCEPTION
            WHEN OTHERS THEN

               BEGIN
                  RAISE NOTICE '%%', 'error compiling ' ,  v_procname;
               END;
         END;
         FETCH v_proclist INTO v_procname;
      END;
   END LOOP;
   -- while loop
   CLOSE v_proclist;-- procedure
END;
 
$$;


ALTER FUNCTION tm_cz.util_recompile_all() OWNER TO tm_cz;

--
-- Name: util_truncate_table(character varying, character varying); Type: FUNCTION; Schema: tm_cz; Owner: tm_cz
--

CREATE FUNCTION tm_cz.util_truncate_table(v_tabname character varying DEFAULT NULL::character varying, v_dummyarg character varying DEFAULT 'Y'::character varying) RETURNS void
    LANGUAGE plpgsql
    AS $$
DECLARE

-------------------------------------------------------------------------------------
-- NAME: UTIL_TRUNCATE_TABLE
--
-- Copyright c 2011 Recombinant Data Corp.
--

--------------------------------------------------------------------------------------
   v_cmdlist CURSOR IS
     /* disable all foreign key constraints on this table */
     PERFORM 'alter table ' || t1.owner ||'.' || t1.table_name ||' disable constraint '|| t1.constraint_name || '' cmd
     FROM user_constraints t1, user_constraints t2
     WHERE T1.CONSTRAINT_TYPE='R' and T1.R_CONSTRAINT_NAME=T2.CONSTRAINT_NAME and
        T2.TABLE_NAME = v_tabname
     UNION ALL
     /* finally actually truncate the table */
     PERFORM 'truncate table ' || v_tabname || '' cmd
     
     UNION ALL
     /* do a delete just incase the truncate failed */
     PERFORM 'delete from ' || v_tabname || '' cmd
     ;

   v_cmdline varchar(200);
   v_drop_if_populated integer;


BEGIN

   /* done with the SQL select - now process each command we selected */
   OPEN v_cmdlist;
   FETCH v_cmdlist INTO v_cmdline;
   WHILE v_cmdlist%FOUND
   LOOP

      BEGIN
         BEGIN

            BEGIN
               EXECUTE v_cmdline;
               RAISE NOTICE '%%', 'SUCCESS ' ,  v_cmdline;
            END;
         EXCEPTION
            WHEN OTHERS THEN

               BEGIN
                  RAISE NOTICE '%%', 'ERROR ' ,  v_cmdline;
               END;
         END;
         FETCH v_cmdlist INTO v_cmdline;
      END;
   END LOOP;
   -- while loop
   CLOSE v_cmdlist;-- procedure
END;
 
$$;


ALTER FUNCTION tm_cz.util_truncate_table(v_tabname character varying, v_dummyarg character varying) OWNER TO tm_cz;

--
-- Name: i2b2_process_qpcr_mirna_data(character varying, character varying, character varying, character varying, character varying, numeric, character varying, numeric); Type: FUNCTION; Schema: tm_lz; Owner: tm_lz
--

CREATE FUNCTION tm_lz.i2b2_process_qpcr_mirna_data(trial_id character varying, top_node character varying, mirna_type character varying, data_type character varying DEFAULT 'R'::character varying, source_cd character varying DEFAULT 'STD'::character varying, log_base numeric DEFAULT 2, secure_study character varying DEFAULT NULL::character varying, currentjobid numeric DEFAULT '-1'::integer, OUT rtn_code numeric) RETURNS numeric
    LANGUAGE plpgsql
    AS $_$
DECLARE

/*************************************************************************

* This store procedure is for ETL for Sanofi to load  qpcr or seq miRNA data
* Date: 12/05/2013

******************************************************************/









--	***  NOTE ***
--	The input file columns are mapped to the following table columns.  This is done so that the javascript for the advanced workflows
--	selects the correct data for the dropdowns.

--		tissue_type	=>	sample_type
--		attribute_1	=>	tissue_type
--		atrribute_2	=>	timepoint	

  TrialID		varchar(100);
  RootNode		varchar(2000);
  root_level	integer;
  topNode		varchar(2000);
  topLevel		integer;
  tPath			varchar(2000);
  study_name	varchar(100);
  sourceCd		varchar(50);
  secureStudy	varchar(1);

  dataType		varchar(10);
  sqlText		varchar(1000);
  tText			varchar(1000);
  gplTitle		varchar(1000);
  pExists		bigint;
  partTbl   	bigint;
  partExists 	bigint;
  sampleCt		bigint;
  idxExists 	bigint;
  logBase		bigint;
  pCount		integer;
  sCount		integer;
  tablespaceName	varchar(200);
  v_bio_experiment_id	bigint;
  mirnaType varchar(15);
 -- mirnaPlatform varchar2(20);
  
    --Audit variables
  newJobFlag integer(1);
  databaseName varchar(100);
  procedureName varchar(100);
  jobID bigint;
  stepCt bigint;
  
  --unmapped_patients exception;
  missing_platform	exception;
  missing_tissue	EXCEPTION;
  unmapped_platform exception;
  multiple_platform	exception;
  no_probeset_recs	exception;
 -- missing_mirna_type	EXCEPTION;
  

  
	addNodes CURSOR FOR
	SELECT distinct t.leaf_node
          ,t.node_name
	from  wt_qpcr_mirna_nodes t
	where not exists
		 (select 1 from i2b2 x
		  where t.leaf_node = x.c_fullname);

 
--	cursor to define the path for delete_one_node  this will delete any nodes that are hidden after i2b2_create_concept_counts

  delNodes CURSOR FOR
  SELECT distinct c_fullname 
  from  i2b2
  where c_fullname like topNode || '%'
    and substring(c_visualattributes from 2 for 1) = 'H';
    --and c_visualattributes like '_H_';



BEGIN
	TrialID := upper(trial_id);
	secureStudy := upper(secure_study);
	mirnaType:=upper(mirna_type);
	
	if (secureStudy not in ('Y','N') ) then
		secureStudy := 'Y';
	end if;
	
	topNode := REGEXP_REPLACE('\' || top_node || '\','(\\){2,}', '\');	
	PERFORM length(topNode)-length(replace(topNode,'\','')) into topLevel ;
	
	if coalesce(data_type::text, '') = '' then
		dataType := 'R';
	else
		if data_type in ('R','T','L') then
			dataType := data_type;
		else
			dataType := 'R';
		end if;
	end if;

	---check for mirna_type is not null if it is null raise an exception 
	/*if mirna_type is null then
	raise missing_mirna_type;
	end if;
	*/
	logBase := log_base;
	sourceCd := upper(coalesce(source_cd,'STD'));

  --Set Audit Parameters
  newJobFlag := 0; -- False (Default)
  jobID := currentJobID;

  PERFORM sys_context('USERENV', 'CURRENT_SCHEMA') INTO databaseName ;
  procedureName := $$PLSQL_UNIT;

  --Audit JOB Initialization
  --If Job ID does not exist, then this is a single procedure run and we need to create it
  IF(coalesce(jobID::text, '') = '' or jobID < 1)
  THEN
    newJobFlag := 1; -- True
    cz_start_audit (procedureName, databaseName, jobID);
  END IF;
    	
	stepCt := 0;
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Starting i2b2_process_qpcr_mirna_data',0,stepCt,'Done');
	
	--	Get count of records in LT_SRC_MIRNA_SUBJ_SAMP_MAP
	
	select count(*) into sCount
	from LT_SRC_MIRNA_SUBJ_SAMP_MAP;
	
	--	check if all subject_sample map records have a platform, If not, abort run
	
	select count(*) into pCount
	from LT_SRC_MIRNA_SUBJ_SAMP_MAP
	where coalesce(platform::text, '') = '';
	
	if pCount > 0 then
		raise missing_platform;
	end if;
  
  	--	check if platform exists in de_qpcr_mirna_annotation .  If not, abort run.
	
	select count(*) into pCount
	from LT_QPCR_MIRNA_ANNOTATION
	where ID_REF in (select distinct m.platform from LT_SRC_MIRNA_SUBJ_SAMP_MAP m);
	
	--if PCOUNT = 0 then
		--RAISE UNMAPPED_platform;
	--end if;--mod
	
	select count(*) into pCount
	from DE_gpl_info
	where platform in (select distinct m.platform from LT_SRC_MIRNA_SUBJ_SAMP_MAP m);
	
	/*if PCOUNT = 0 then
		RAISE UNMAPPED_platform;
	end if;*/
		
	--	check if all subject_sample map records have a tissue_type, If not, abort run
	
	select count(*) into pCount
	from LT_SRC_MIRNA_SUBJ_SAMP_MAP
	where coalesce(tissue_type::text, '') = '';
	
	if pCount > 0 then
		raise missing_tissue;
	end if;
	
	--	check if there are multiple platforms, if yes, then platform must be supplied in LT_SRC_QPCR_MIRNA_DATA
	
	select count(*) into pCount
	from (select sample_cd
		  from LT_SRC_MIRNA_SUBJ_SAMP_MAP
		  group by sample_cd
		  GROUP BY xd.concept_Cd
		  having Max(xf.valtype_cd) = 'N');
		  
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName, HAVING count(distinct platform) > 1);
	
	if pCount > 0 then
		raise multiple_platform;
	end if;
		
	-- Get root_node from topNode
  
	PERFORM parse_nth_value(topNode, 2, '\') into RootNode ;
	
	select count(*) into pExists
	from table_access
	where c_name = rootNode;
	
	if pExists = 0 then
		i2b2_add_root_node(rootNode, jobId);
	end if;
	
	select c_hlevel into root_level
	from i2b2
	where c_name = RootNode;
	
	-- Get study name from topNode
  
	PERFORM parse_nth_value(topNode, topLevel, '\') into study_name ;
	
	--	Add any upper level nodes as needed
	
	tPath := REGEXP_REPLACE(replace(top_node,study_name,null),'(\\){2,}', '\');
	PERFORM length(tPath) - length(replace(tPath,'\',null)) into pCount ;

	if pCount > 2 then
		i2b2_fill_in_tree(null, tPath, jobId);
	end if;

	--	uppercase study_id in lt_src_mirna_subj_samp_map in case curator forgot
	
	update LT_SRC_MIRNA_SUBJ_SAMP_MAP
	set trial_name=upper(trial_name);
	
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Uppercase trial_name in LT_SRC_MIRNA_SUBJ_SAMP_MAP',SQL%ROWCOUNT,stepCt,'Done');
	commit;	
	
	--	create records in patient_dimension for subject_ids if they do not exist
	--	format of sourcesystem_cd:  trial:[site:]subject_cd
	
	insert into patient_dimension
    ( patient_num,
      sex_cd,
      age_in_years_num,
      race_cd,
      update_date,
      download_date,
      import_date,
      sourcesystem_cd
    )
    PERFORM nextval('seq_patient_num')
		  ,x.sex_cd
		  ,x.age_in_years_num
		  ,x.race_cd
		  ,LOCALTIMESTAMP
		  ,LOCALTIMESTAMP
		  ,LOCALTIMESTAMP
		  ,x.sourcesystem_cd
	from (select distinct 'Unknown' as sex_cd,
				 0 as age_in_years_num,
				 null as race_cd,
				 regexp_replace(TrialID || ':' || s.site_id || ':' || s.subject_id,'(::){1,}', ':') as sourcesystem_cd
		 from LT_SRC_MIRNA_SUBJ_SAMP_MAP s
		     ,de_gpl_info g
		 where (s.subject_id IS NOT NULL AND s.subject_id::text <> '')
		   and s.trial_name = TrialID
		   and s.source_cd = sourceCD
		   and s.platform = g.platform
		   and upper(g.marker_type) = mirnaType
		   and not exists
			  (select 1 from patient_dimension x
			   where x.sourcesystem_cd = 
				 regexp_replace(TrialID || ':' || s.site_id || ':' || s.subject_id,'(::){1,}', ':'))
		) x;
	
	pCount := SQL%ROWCOUNT;
	
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Insert subjects to patient_dimension',pCount,stepCt,'Done');
	commit;
	
	i2b2_create_security_for_trial(TrialId, secureStudy, jobID);

	--	Delete existing observation_fact data, will be repopulated
	
	delete from observation_fact obf
	where obf.concept_cd in
		 (select distinct x.concept_code
		  from de_subject_sample_mapping x
		  where x.trial_name = TrialId
		    and coalesce(x.source_cd,'STD') = sourceCD
		    and x.platform = mirna_type);

	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Delete data from observation_fact',SQL%ROWCOUNT,stepCt,'Done');
	commit;

	select count(*) into pExists
	from all_tables
	where table_name = 'DE_SUBJECT_MIRNA_DATA'
	  and partitioned = 'YES';
	
	if pExists = 0 then
		--	dataset is not partitioned so must delete
		
		delete from de_subject_mirna_data
		where trial_source = TrialId || ':' || sourceCd;
		stepCt := stepCt + 1;
		cz_write_audit(jobId,databaseName,procedureName,'Delete data from de_subject_mirna_data',SQL%ROWCOUNT,stepCt,'Done');
		commit;
	else
		--	Create partition in de_subject_MIRNA_data if it doesn't exist else truncate partition
			
		select count(*)
			into pExists
			from all_tab_partitions
			where table_name = 'DE_SUBJECT_MIRNA_DATA'
			  and partition_name = TrialId || ':' || sourceCd;
			
		if pExists = 0 then
					
			--	needed to add partition to de_subject_MIRNA_data

			sqlText := 'alter table deapp.de_subject_mirna_data add PARTITION "' || TrialID || ':' || sourceCd || '"  VALUES (' || '''' || TrialID || ':' || sourceCd || '''' || ') ' ||
						   'NOLOGGING COMPRESS TABLESPACE "TRANSMART" ';
			EXECUTE(sqlText);
			stepCt := stepCt + 1;
			cz_write_audit(jobId,databaseName,procedureName,'Adding partition to de_subject_mirna_data',0,stepCt,'Done');
				
		else
			sqlText := 'alter table deapp.de_subject_mirna_data truncate partition "' || TrialID || ':' || sourceCd || '"';
			EXECUTE(sqlText);
			stepCt := stepCt + 1;
			cz_write_audit(jobId,databaseName,procedureName,'Truncating partition in de_subject_mirna_data',0,stepCt,'Done');
		end if;
		
	end if;
		
	--	Cleanup any existing data in de_subject_sample_mapping.  

	delete from DE_SUBJECT_SAMPLE_MAPPING 
	where trial_name = TrialID 
	  and coalesce(source_cd,'STD') = sourceCd
	  and platform = mirna_type; --Making sure only miRNA data is deleted
		  
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Delete trial from DEAPP de_subject_sample_mapping',SQL%ROWCOUNT,stepCt,'Done');

	commit;

--	truncate tmp node table

	EXECUTE('truncate table tm_wz.WT_QPCR_MIRNA_NODES');
	
--	load temp table with leaf node path, use temp table with distinct sample_type, ATTR2, platform, and title   this was faster than doing subselect
--	from wt_subject_mirna_data

	EXECUTE('truncate table tm_wz.WT_QPCR_MIRNA_NODE_VALUES');
	
	insert into WT_QPCR_MIRNA_NODE_VALUES
	(category_cd
	,platform
	,tissue_type
	,attribute_1
	,attribute_2
	,title
	)
	PERFORM distinct a.category_cd
				   ,coalesce(a.platform,'GPL570')
				   ,coalesce(a.tissue_type,'Unspecified Tissue Type')
	               ,a.attribute_1
				   ,a.attribute_2
				   ,g.title
    from LT_SRC_MIRNA_SUBJ_SAMP_MAP a
	    ,de_gpl_info g 
	where a.trial_name = TrialID
	  and coalesce(a.platform,'GPL570') = g.platform
	  and a.source_cd = sourceCD
	  and a.platform = g.platform
	  and upper(g.marker_type) = mirnaType
	  and g.title = (select min(x.title) from de_gpl_info x where coalesce(a.platform,'GPL570') = x.platform)
      -- and upper(g.organism) = 'HOMO SAPIENS'
	  ;
        
	--  and decode(dataType,'R',sign(a.intensity_value),1) = 1;	--	take all values when dataType T, only >0 for dataType R
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Insert node values into DEAPP wt_qpcr_mirna_node_values',SQL%ROWCOUNT,stepCt,'Done');
	commit;
	
	insert into WT_QPCR_MIRNA_NODES
	(leaf_node
	,category_cd
	,platform
	,tissue_type
	,attribute_1
    ,attribute_2
	,node_type
	)
	PERFORM distinct topNode || regexp_replace(replace(replace(replace(replace(replace(replace(
	       category_cd,'PLATFORM',title),'ATTR1',attribute_1),'ATTR2',attribute_2),'TISSUETYPE',tissue_type),'+','\'),'_',' ') || '\','(\\){2,}', '\') 
		  ,category_cd
		  ,platform as platform
		  ,tissue_type
		  ,attribute_1 as attribute_1
          ,attribute_2 as attribute_2
		  ,'LEAF'
	from  WT_QPCR_MIRNA_NODE_VALUES;
		   
    stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Create leaf nodes in DEAPP tmp_mirna_nodes',SQL%ROWCOUNT,stepCt,'Done');
	commit;
	
	--	insert for platform node so platform concept can be populated
	
	insert into WT_QPCR_MIRNA_NODES
	(leaf_node
	,category_cd
	,platform
	,tissue_type
	,attribute_1
    ,attribute_2
	,node_type
	)
	PERFORM distinct topNode || regexp_replace(replace(replace(replace(replace(replace(replace(
	       substr(category_cd,1,instr(category_cd,'PLATFORM')+8),'PLATFORM',title),'ATTR1',attribute_1),'ATTR2',attribute_2),'TISSUETYPE',tissue_type),'+','\'),'_',' ') || '\',
		   '(\\){2,}', '\')
		  ,substr(category_cd,1,instr(category_cd,'PLATFORM')+8)
		  ,platform as platform
		  ,case when instr(substr(category_cd,1,instr(category_cd,'PLATFORM')+8),'TISSUETYPE') > 1 then tissue_type else null end as tissue_type
		  ,case when instr(substr(category_cd,1,instr(category_cd,'PLATFORM')+8),'ATTR1') > 1 then attribute_1 else null end as attribute_1
          ,case when instr(substr(category_cd,1,instr(category_cd,'PLATFORM')+8),'ATTR2') > 1 then attribute_2 else null end as attribute_2
		  ,'PLATFORM'
	from  WT_QPCR_MIRNA_NODE_VALUES;
		   
    stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Create platform nodes in wt_qpcr_mirna_nodes',SQL%ROWCOUNT,stepCt,'Done');
	commit;
	
	--	insert for ATTR1 node so ATTR1 concept can be populated in tissue_type_cd
	
	insert into WT_QPCR_MIRNA_NODES
	(leaf_node
	,category_cd
	,platform
	,tissue_type
    ,attribute_1
	,attribute_2
	,node_type
	)
	PERFORM distinct topNode || regexp_replace(replace(replace(replace(replace(replace(replace(
	       substr(category_cd,1,instr(category_cd,'ATTR1')+5),'PLATFORM',title),'ATTR1',attribute_1),'ATTR2',attribute_2),'TISSUETYPE',tissue_type),'+','\'),'_',' ') || '\',
		   '(\\){2,}', '\')
		  ,substr(category_cd,1,instr(category_cd,'ATTR1')+5)
		  ,case when instr(substr(category_cd,1,instr(category_cd,'ATTR1')+5),'PLATFORM') > 1 then platform else null end as platform
		  ,case when instr(substr(category_cd,1,instr(category_cd,'ATTR1')+5),'TISSUETYPE') > 1 then tissue_type else null end as tissue_type
		  ,attribute_1 as attribute_1
          ,case when instr(substr(category_cd,1,instr(category_cd,'ATTR1')+5),'ATTR2') > 1 then attribute_2 else null end as attribute_2
		  ,'ATTR1'
	from  WT_QPCR_MIRNA_NODE_VALUES
	where category_cd like '%ATTR1%'
	  and (attribute_1 IS NOT NULL AND attribute_1::text <> '');
		   
    stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Create ATTR1 nodes in WT_QPCR_MIRNA_NODES',SQL%ROWCOUNT,stepCt,'Done');
	commit;
	
	--	insert for ATTR2 node so ATTR2 concept can be populated in timepoint_cd
	
	insert into WT_QPCR_MIRNA_NODES
	(leaf_node
	,category_cd
	,platform
	,tissue_type
    ,attribute_1
	,attribute_2
	,node_type
	)
	PERFORM distinct topNode || regexp_replace(replace(replace(replace(replace(replace(replace(
	       substr(category_cd,1,instr(category_cd,'ATTR2')+5),'PLATFORM',title),'ATTR1',attribute_1),'ATTR2',attribute_2),'TISSUETYPE',tissue_type),'+','\'),'_',' ') || '\',
		   '(\\){2,}', '\')
		  ,substr(category_cd,1,instr(category_cd,'ATTR2')+5)
		  ,case when instr(substr(category_cd,1,instr(category_cd,'ATTR2')+5),'PLATFORM') > 1 then platform else null end as platform
		  ,case when instr(substr(category_cd,1,instr(category_cd,'ATTR1')+5),'TISSUETYPE') > 1 then tissue_type else null end as tissue_type
          ,case when instr(substr(category_cd,1,instr(category_cd,'ATTR2')+5),'ATTR1') > 1 then attribute_1 else null end as attribute_1
		  ,attribute_2 as attribute_2
		  ,'ATTR2'
	from  WT_QPCR_MIRNA_NODE_VALUES
	where category_cd like '%ATTR2%'
	  and (attribute_2 IS NOT NULL AND attribute_2::text <> '');
		   
    stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Create ATTR2 nodes in WT_QPCR_MIRNA_NODES',SQL%ROWCOUNT,stepCt,'Done');
	commit;
	
	--	insert for tissue_type node so sample_type_cd can be populated

	insert into WT_QPCR_MIRNA_NODES
	(leaf_node
	,category_cd
	,platform
	,tissue_type
	,attribute_1
    ,attribute_2
	,node_type
	)
	PERFORM distinct topNode || regexp_replace(replace(replace(replace(replace(replace(replace(
	       substr(category_cd,1,instr(category_cd,'TISSUETYPE')+10),'PLATFORM',title),'ATTR1',attribute_1),'ATTR2',attribute_2),'TISSUETYPE',tissue_type),'+','\'),'_',' ') || '\',
		   '(\\){2,}', '\')
		  ,substr(category_cd,1,instr(category_cd,'TISSUETYPE')+10)
		  ,case when instr(substr(category_cd,1,instr(category_cd,'TISSUETYPE')+10),'PLATFORM') > 1 then platform else null end as platform
		  ,tissue_type as tissue_type
		  ,case when instr(substr(category_cd,1,instr(category_cd,'TISSUETYPE')+10),'ATTR1') > 1 then attribute_1 else null end as attribute_1
          ,case when instr(substr(category_cd,1,instr(category_cd,'TISSUETYPE')+10),'ATTR2') > 1 then attribute_2 else null end as attribute_2
		  ,'TISSUETYPE'
	from  WT_QPCR_MIRNA_NODE_VALUES
	where category_cd like '%TISSUETYPE%';
		   
    stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Create ATTR2 nodes in wt_qpcr_mirna_nodes',SQL%ROWCOUNT,stepCt,'Done');
	commit;
				
	update WT_QPCR_MIRNA_NODES
	set node_name=parse_nth_value(leaf_node,length(leaf_node)-length(replace(leaf_node,'\',null)),'\');
		   
    stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Updated node_name in DEAPP tmp_mirna_nodes',SQL%ROWCOUNT,stepCt,'Done');
	commit;
		
--	add leaf nodes for miRNA data  The cursor will only add nodes that do not already exist.

	 FOR r_addNodes in addNodes Loop

    --Add nodes for all types (ALSO DELETES EXISTING NODE)

		i2b2_add_node(TrialID, r_addNodes.leaf_node, r_addNodes.node_name, jobId);
		stepCt := stepCt + 1;
		tText := 'Added Leaf Node: ' || r_addNodes.leaf_node || '  Name: ' || r_addNodes.node_name;
		
		cz_write_audit(jobId,databaseName,procedureName,tText,SQL%ROWCOUNT,stepCt,'Done');
		
		i2b2_fill_in_tree(TrialId, r_addNodes.leaf_node, jobID);

	END LOOP;  
	
--	update concept_cd for nodes, this is done to make the next insert easier
	
	update WT_QPCR_MIRNA_NODES t
	set concept_cd=(select c.concept_cd from concept_dimension c
	                where c.concept_path = t.leaf_node LIMIT 1 OFFSET 1
				   )
    where exists
         (select 1 from concept_dimension x
	                where x.concept_path = t.leaf_node
				   )
	  and coalesce(t.concept_cd::text, '') = '';
	
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Update WT_QPCR_MIRNA_NODES with newly created concept_cds',SQL%ROWCOUNT,stepCt,'Done');
	commit;
	 
	
  --Load the DE_SUBJECT_SAMPLE_MAPPING from wt_subject_mirna_data

  --PATIENT_ID      = PATIENT_ID (SAME AS ID ON THE PATIENT_DIMENSION)
  --SITE_ID         = site_id
  --SUBJECT_ID      = subject_id
  --SUBJECT_TYPE    = NULL
  --CONCEPT_CODE    = from LEAF records in wt_mirna_nodes
  --SAMPLE_TYPE    	= TISSUE_TYPE
  --SAMPLE_TYPE_CD  = concept_cd from TISSUETYPE records in wt_mirna_nodes
  --TRIAL_NAME      = TRIAL_NAME
  --TIMEPOINT		= attribute_2
  --TIMEPOINT_CD	= concept_cd from ATTR2 records in wt_mirna_nodes
  --TISSUE_TYPE     = attribute_1
  --TISSUE_TYPE_CD  = concept_cd from ATTR1 records in wt_mirna_nodes
  --PLATFORM        = MIRNA_AFFYMETRIX - this is required by ui code
  --PLATFORM_CD     = concept_cd from PLATFORM records in wt_qpcr_mirna_nodes
  --DATA_UID		= concatenation of concept_cd-patient_num
  --GPL_ID			= platform from wt_subject_mirna_data
  --CATEGORY_CD		= category_cd that generated ontology
  --SAMPLE_ID		= id of sample (trial:S:[site_id]:subject_id:sample_cd) from patient_dimension, may be the same as patient_num
  --SAMPLE_CD		= sample_cd
  --SOURCE_CD		= sourceCd
  
  --ASSAY_ID        = generated by trigger

	insert into de_subject_sample_mapping
	(patient_id
	,site_id
	,subject_id
	,subject_type
	,concept_code
	,assay_id
	,sample_type
	,sample_type_cd
	,trial_name
	,timepoint
	,timepoint_cd
	,tissue_type
	,tissue_type_cd
	,platform
	,platform_cd
	,data_uid
	,gpl_id
	,sample_id
	,sample_cd
	,category_cd
	,source_cd
	,omic_source_study
	,omic_patient_id
    )
	PERFORM t.patient_id
		  ,t.site_id
		  ,t.subject_id
		  ,t.subject_type
		  ,t.concept_code
		  ,deapp.nextval('seq_assay_id')
		  ,t.sample_type
		  ,t.sample_type_cd
		  ,t.trial_name
		  ,t.timepoint
		  ,t.timepoint_cd
		  ,t.tissue_type
		  ,t.tissue_type_cd
		  ,t.platform
		  ,t.platform_cd
		  ,t.data_uid
		  ,t.gpl_id
		  ,t.sample_id
		  ,t.sample_cd
		  ,t.category_cd
		  ,t.source_cd
		  ,t.omic_source_study
		  ,t.omic_patient_id
	from (select distinct b.patient_num as patient_id
			  ,a.site_id
			  ,a.subject_id
			  ,null as subject_type
			  ,ln.concept_cd as concept_code
			  ,a.tissue_type as sample_type
			  ,ttp.concept_cd as sample_type_cd
			  ,a.trial_name
			  ,a.attribute_2 as timepoint
			  ,a2.concept_cd as timepoint_cd
			  ,a.attribute_1 as tissue_type
			  ,a1.concept_cd as tissue_type_cd
			  ,mirna_type as platform
			  ,pn.concept_cd as platform_cd
			  ,ln.concept_cd || '-' || to_char(b.patient_num) as data_uid
			  ,a.platform as gpl_id
			  ,coalesce(sid.patient_num,b.patient_num) as sample_id
			  ,a.sample_cd
			  ,coalesce(a.category_cd,'Biomarker_Data+QPCR_MIRNA+PLATFORM+TISSUETYPE+ATTR1+ATTR2') as category_cd
			  ,a.source_cd
			  ,TrialId as omic_source_study
			  ,b.patient_num as omic_patient_id
		from lt_src_mirna_subj_samp_map a		
		--Joining to Pat_dim to ensure the ID's match. If not I2B2 won't work.
		inner join patient_dimension b
		  on regexp_replace(TrialID || ':' || a.site_id || ':' || a.subject_id,'(::){1,}', ':') = b.sourcesystem_cd
		inner join WT_QPCR_MIRNA_NODES ln
			on a.platform = ln.platform
			and a.tissue_type = ln.tissue_type
			and coalesce(a.attribute_1,'@') = coalesce(ln.attribute_1,'@')
			and coalesce(a.attribute_2,'@') = coalesce(ln.attribute_2,'@')
			and ln.node_type = 'LEAF'
		inner join WT_QPCR_MIRNA_NODES pn
			on a.platform = pn.platform
			and case when instr(substr(a.category_cd,1,instr(a.category_cd,'PLATFORM')+8),'TISSUETYPE') > 1 then a.tissue_type else '@' end = coalesce(pn.tissue_type,'@')
			and case when instr(substr(a.category_cd,1,instr(a.category_cd,'PLATFORM')+8),'ATTR1') > 1 then a.attribute_1 else '@' end = coalesce(pn.attribute_1,'@')
			and case when instr(substr(a.category_cd,1,instr(a.category_cd,'PLATFORM')+8),'ATTR2') > 1 then a.attribute_2 else '@' end = coalesce(pn.attribute_2,'@')
			and pn.node_type = 'PLATFORM'	  
		left outer join WT_QPCR_MIRNA_NODES ttp
			on a.tissue_type = ttp.tissue_type
			and case when instr(substr(a.category_cd,1,instr(a.category_cd,'TISSUETYPE')+10),'PLATFORM') > 1 then a.platform else '@' end = coalesce(ttp.platform,'@')
			and case when instr(substr(a.category_cd,1,instr(a.category_cd,'TISSUETYPE')+10),'ATTR1') > 1 then a.attribute_1 else '@' end = coalesce(ttp.attribute_1,'@')
			and case when instr(substr(a.category_cd,1,instr(a.category_cd,'TISSUETYPE')+10),'ATTR2') > 1 then a.attribute_2 else '@' end = coalesce(ttp.attribute_2,'@')
			and ttp.node_type = 'TISSUETYPE'		  
		left outer join WT_QPCR_MIRNA_NODES a1
			on a.attribute_1 = a1.attribute_1
			and case when instr(substr(a.category_cd,1,instr(a.category_cd,'ATTR1')+5),'PLATFORM') > 1 then a.platform else '@' end = coalesce(a1.platform,'@')
			and case when instr(substr(a.category_cd,1,instr(a.category_cd,'ATTR1')+5),'TISSUETYPE') > 1 then a.tissue_type else '@' end = coalesce(a1.tissue_type,'@')
			and case when instr(substr(a.category_cd,1,instr(a.category_cd,'ATTR1')+5),'ATTR2') > 1 then a.attribute_2 else '@' end = coalesce(a1.attribute_2,'@')
			and a1.node_type = 'ATTR1'		  
		left outer join WT_QPCR_MIRNA_NODES a2
			on a.attribute_2 = a1.attribute_2
			and case when instr(substr(a.category_cd,1,instr(a.category_cd,'ATTR2')+5),'PLATFORM') > 1 then a.platform else '@' end = coalesce(a2.platform,'@')
			and case when instr(substr(a.category_cd,1,instr(a.category_cd,'ATTR2')+5),'TISSUETYPE') > 1 then a.tissue_type else '@' end = coalesce(a2.tissue_type,'@')
			and case when instr(substr(a.category_cd,1,instr(a.category_cd,'ATTR2')+5),'ATTR1') > 1 then a.attribute_1 else '@' end = coalesce(a2.attribute_1,'@')
			and a2.node_type = 'ATTR2'			  
		left outer join patient_dimension sid
			on  regexp_replace(TrialId || 'S:' || a.site_id || ':' || a.subject_id || ':' || a.sample_cd,
							  '(::){1,}', ':') = sid.sourcesystem_cd
		where a.trial_name = TrialID
		  and a.source_cd = sourceCD
		  and  (ln.concept_cd IS NOT NULL AND ln.concept_cd::text <> '')) t;

	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Insert trial into DEAPP de_subject_sample_mapping',SQL%ROWCOUNT,stepCt,'Done');

  commit;

--	recreate de_subject_sam
--	recreate de_subject_sample_mapping indexes

	--execute immediate('create index de_subject_smpl_mpng_idx1 on de_subject_sample_mapping(timepoint, patient_id, trial_name) parallel nologging'); 
	--execute immediate('create index de_subject_smpl_mpng_idx2 on de_subject_sample_mapping(patient_id, timepoint_cd, platform_cd, assay_id, trial_name) parallel nologging'); 
	--execute immediate('create bitmap index de_subject_smpl_mpng_idx3 on de_subject_sample_mapping(sample_type_cd) parallel nologging');
	--execute immediate('create index de_subject_smpl_mpng_idx4 on de_subject_sample_mapping(gpl_id) parallel nologging');
	--execute immediate('create index de_subject_smpl_mpng_idx4 on de_subject_sample_mapping(platform, gpl_id) parallel nologging');
    --stepCt := stepCt + 1;
	--cz_write_audit(jobId,databaseName,procedureName,'Recreate indexes on DEAPP de_subject_sample_mapping',0,stepCt,'Done');

--	Insert records for patients and samples into observation_fact

	insert into observation_fact
    (patient_num
	,concept_cd
	,modifier_cd
	,valtype_cd
	,tval_char
	,nval_num
	,sourcesystem_cd
	,import_date
	,valueflag_cd
	,provider_id
	,location_cd
	,units_cd
        ,INSTANCE_NUM
    )
    PERFORM distinct m.patient_id
		  ,m.concept_code
		  ,'@'
		  ,'T' -- Text data type
		  ,'E'  --Stands for Equals for Text Types
		  ,null	--	not numeric for qpcr_mirna
		  ,m.trial_name
		  ,LOCALTIMESTAMP
		  ,'@'
		  ,'@'
		  ,'@'
		  ,'' -- no units available
                  ,1
    from  de_subject_sample_mapping m
    where m.trial_name = TrialID 
	  and m.source_cd = sourceCD
      and m.platform = mirna_type;
	  
    stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Insert patient facts into I2B2DEMODATA observation_fact',SQL%ROWCOUNT,stepCt,'Done');

    commit;
    
	--	Insert sample facts 
	
	insert into observation_fact
    (patient_num
	,concept_cd
	,modifier_cd
	,valtype_cd
	,tval_char
	,nval_num
	,sourcesystem_cd
	,import_date
	,valueflag_cd
	,provider_id
	,location_cd
	,units_cd
    )
    PERFORM distinct m.sample_id
		  ,m.concept_code
		  ,m.trial_name
		  ,'T' -- Text data type
		  ,'E'  --Stands for Equals for Text Types
		  ,null	--	not numeric for miRNA
		  ,m.trial_name
		  ,LOCALTIMESTAMP
		  ,'@'
		  ,'@'
		  ,'@'
		  ,'' -- no units available
    from  de_subject_sample_mapping m
    where m.trial_name = TrialID 
	  and m.source_cd = sourceCd
      and m.platform = mirna_type
	 and m.patient_id != m.sample_id;
	  
    stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Insert sample facts into I2B2DEMODATA observation_fact',SQL%ROWCOUNT,stepCt,'Done');

    commit;
    
	--Update I2b2 for correct data type
	
	update i2b2 t
	set c_columndatatype = 'T', c_metadataxml = null, c_visualattributes='FA'
	where t.c_basecode in (select distinct x.concept_cd from WT_QPCR_MIRNA_NODES x);
  
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Initialize data_type and xml in i2b2',SQL%ROWCOUNT,stepCt,'Done');
	commit;
	
	update i2b2
	SET c_columndatatype = 'N',
      --Static XML String
		c_metadataxml = '<?xml version="1.0"?><ValueMetadata><Version>3.02</Version><CreationDateTime>10/21/2013 01:22:59</CreationDateTime><TestID></TestID><TestName></TestName><DataType>PosFloat</DataType><CodeType></CodeType><Loinc></Loinc><Flagstouse></Flagstouse><Oktousevalues>Y</Oktousevalues><MaxStringLength></MaxStringLength><LowofLowValue>0</LowofLowValue><HighofLowValue>0</HighofLowValue><LowofHighValue>100</LowofHighValue>100<HighofHighValue>100</HighofHighValue><LowofToxicValue></LowofToxicValue><HighofToxicValue></HighofToxicValue><EnumValues></EnumValues><CommentsDeterminingExclusion><Com></Com></CommentsDeterminingExclusion><UnitValues><NormalUnits>ratio</NormalUnits><EqualUnits></EqualUnits><ExcludingUnits></ExcludingUnits><ConvertingUnits><Units></Units><MultiplyingFactor></MultiplyingFactor></ConvertingUnits></UnitValues><Analysis><Enums /><Counts /><New /></Analysis></ValueMetadata>'
	where c_basecode IN (
		  SELECT xd.concept_cd
		  from WT_QPCR_MIRNA_NODES xd
			  ,observation_fact xf
		  where xf.concept_cd = xd.concept_cd
		  procedureName,'Update c_columndatatype and c_metadataxml for numeric data types in I2B2METADATA i2b2',SQL%ROWCOUNT,stepCt,'Done');
	commit;
 
/*
	--UPDATE VISUAL ATTRIBUTES for Leaf Active (Default is folder)
	update i2b2 a
    set c_visualattributes = 'LA'
    where 1 = (
      select count(*)
      from i2b2 b
      where b.c_fullname like (a.c_fullname || '%'))
      and c_fullname like '%' || topNode || '%';
*/

	--UPDATE VISUAL ATTRIBUTES for Leaf Active (Default is folder)
	update i2b2 a
    set c_visualattributes = 'LAH'
	where a.c_basecode in (select distinct x.concept_code from de_subject_sample_mapping x
						   where x.trial_name = TrialId
						     and x.platform = mirna_type
							 and (x.concept_code IS NOT NULL AND x.concept_code::text <> ''));
	  
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Update visual attributes for leaf nodes in I2B2METADATA i2b2',SQL%ROWCOUNT,stepCt,'Done');
  
	COMMIT;
  
  --Build concept Counts
  --Also marks any i2B2 records with no underlying data as Hidden, need to do at Trial level because there may be multiple platform and there is no longer
  -- a unique top-level node for miRNA data
  
    i2b2_create_concept_counts(topNode ,jobID );
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Create concept counts',0,stepCt,'Done');
	
	--	delete each node that is hidden
	
	 FOR r_delNodes in delNodes Loop

    --	deletes hidden nodes for a trial one at a time

		i2b2_delete_1_node(r_delNodes.c_fullname);
		stepCt := stepCt + 1;
		tText := 'Deleted node: ' || r_delNodes.c_fullname;
		
		cz_write_audit(jobId,databaseName,procedureName,tText,SQL%ROWCOUNT,stepCt,'Done');

	END LOOP;  	


  --Reload Security: Inserts one record for every I2B2 record into the security table

    i2b2_load_security_data(jobId);
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Load security data',0,stepCt,'Done');

--	tag data with probeset_id from reference.probeset_deapp
  
	EXECUTE ('truncate table tm_wz.WT_SUBJECT_MIRNA_PROBESET');
	
	--	note: assay_id represents a unique subject/site/sample
	
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'TrialId:'||TrialID,0,stepCt,'test');
  
	insert into WT_SUBJECT_MIRNA_PROBESET  --mod
	(probeset_id
--	,expr_id
	,intensity_value
	,patient_id
--	,sample_cd
--	,subject_id
	,trial_name
	,assay_id
	)
	PERFORM    p.probeset_id 
		  ,avg(md.intensity_value)
                  ,sd.patient_id
		  ,TrialId
		  ,sd.assay_id
	from deapp.de_subject_sample_mapping sd
		,LT_SRC_QPCR_MIRNA_DATA md   
                ,mirna_probeset_deapp p
	where sd.sample_cd = md.expr_id
	  and sd.platform = 'MIRNA_AFFYMETRIX'
	  and sd.trial_name =TrialId
	  and sd.source_cd = sourceCd
	 -- and sd.gpl_id = gs.id_ref
	  and md.probeset =p.probeset-- gs.mirna_id
	 and decode(dataType,'R',sign(md.intensity_value),1) = 1  
	group by  p.probeset_id 
		  ,sd.patient_id,sd.assay_id;
		  
	pExists := SQL%ROWCOUNT;
	
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'Insert into DEAPP wt_subject_mirna_probeset',SQL%ROWCOUNT,stepCt,'Done');
	
	commit;		
	
	/*if pExists = 0 then
		raise no_probeset_recs;
	end if;*/
--mod

	--	insert into de_subject_mirna_data when dataType is T (transformed)
 
	if dataType = 'T' then
		insert into de_subject_mirna_data
		(trial_source
		,probeset_id
		,assay_id
		,patient_id
		--,sample_id
		--,subject_id
		,trial_name
		,zscore
		)
		PERFORM (TrialId || ':' || sourceCd)
			  ,probeset_id
			  ,assay_id
			  ,patient_id
			  --,sample_id
			  --,subject_id
			  ,trial_name
			  ,case when intensity_value < -2.5
			        then -2.5
					when intensity_value > 2.5
					then 2.5
					else intensity_value
			   end as zscore
		from WT_SUBJECT_MIRNA_PROBESET --mod
		where trial_name = TrialID;
		stepCt := stepCt + 1;
		cz_write_audit(jobId,databaseName,procedureName,'Insert transformed into DEAPP de_subject_mirna_data',SQL%ROWCOUNT,stepCt,'Done');

		commit;	
	else
		
	--	Calculate ZScores and insert data into de_subject_mirna_data.  The 'L' parameter indicates that the gene expression data will be selected from
	--	wt_subject_mirna_probeset as part of a Load.  

		if dataType = 'R' or dataType = 'L' then
			i2b2_mirna_zscore_calc(TrialID,'L',jobId,dataType,logBase,sourceCD);
			stepCt := stepCt + 1;
			cz_write_audit(jobId,databaseName,procedureName,'Calculate Z-Score',0,stepCt,'Done');
			commit;
		end if;
	
	end if;

    ---Cleanup OVERALL JOB if this proc is being run standalone
	
	stepCt := stepCt + 1;
	cz_write_audit(jobId,databaseName,procedureName,'End i2b2_process_QPCR_miRNA_DATA',0,stepCt,'Done');

	IF newJobFlag = 1
	THEN
		cz_end_audit (jobID, 'SUCCESS');
	END IF;
	
	PERFORM 0 into rtn_code ;

	EXCEPTION
	--when unmapped_patients then
	--	cz_write_audit(jobId,databasename,procedurename,'No site_id/subject_id mapped to patient_dimension',1,stepCt,'ERROR');
	--	cz_error_handler(jobid,procedurename);
	--	cz_end_audit (jobId,'FAIL');
	when missing_platform then
		cz_write_audit(jobId,databasename,procedurename,'Platform data missing from one or more subject_sample mapping records',1,stepCt,'ERROR');
		cz_error_handler(jobid,procedurename);
		cz_end_audit (jobId,'FAIL');
		PERFORM 161 into rtn_code ;
	when missing_tissue then
		cz_write_audit(jobId,databasename,procedurename,'Tissue Type data missing from one or more subject_sample mapping records',1,stepCt,'ERROR');
		cz_error_handler(jobid,procedurename);
		CZ_END_AUDIT (JOBID,'FAIL');
		PERFORM 162 into rtn_code ;
	when unmapped_platform then
		cz_write_audit(jobId,databasename,procedurename,'Platform not found in de_qpcr_mirna_annotation',1,stepCt,'ERROR');
		CZ_ERROR_HANDLER(JOBID,PROCEDURENAME);
		cz_end_audit (jobId,'FAIL');
		PERFORM 163 into rtn_code ;--mod
	when multiple_platform then
		cz_write_audit(jobId,databasename,procedurename,'Multiple platforms for sample_cd in LT_SRC_MIRNA_SUBJ_SAMP_MAP',1,stepCt,'ERROR');
		CZ_ERROR_HANDLER(JOBID,PROCEDURENAME);
		cz_end_audit (jobId,'FAIL');
		PERFORM 164 into rtn_code ;
	when no_probeset_recs then
		cz_write_audit(jobId,databasename,procedurename,'Unable to match probesets to platform in probeset_deapp',1,stepCt,'ERROR');
		CZ_ERROR_HANDLER(JOBID,PROCEDURENAME);
		cz_end_audit (jobId,'FAIL');
		PERFORM 165 into rtn_code ;
	WHEN OTHERS THEN
		--Handle errors.
		cz_error_handler (jobID, procedureName);
		--End Proc
		cz_end_audit (jobID, 'FAIL');
		PERFORM 16  into rtn_code ;
END;
 
$_$;


ALTER FUNCTION tm_lz.i2b2_process_qpcr_mirna_data(trial_id character varying, top_node character varying, mirna_type character varying, data_type character varying, source_cd character varying, log_base numeric, secure_study character varying, currentjobid numeric, OUT rtn_code numeric) OWNER TO tm_lz;

--
