dheepa/mmt_sql
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

sql files

Browse Source
This commit is contained in:
dheepa
2021-07-02 08:40:37 +00:00
commit a0d3257be0
154 changed files with 49773 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

142
onetime/commonfunctions/fn_run_post_generic_validation.sql Executable file
View File

@@ -0,0 +1,142 @@
drop function if exists fw_core.fn_run_post_generic_validation;
CREATE OR REPLACE FUNCTION fw_core.fn_run_post_generic_validation(p_client_id int,p_function_id int,p_job_id int,p_step_id int)
RETURNS void AS $$
declare
f record;
trx_record_count int;
keyword_match_count int;
missing_sheet_count int;
success_sum int;
w_job_id int := p_job_id;
w_step_id int := p_step_id;
-- validate file already loaded in the Transactional DB :
begin
for f in select distinct file_syspk from fw_core.fw_jobctl_file_runschedule
where latest_runschedule_flag = '1' and end_status != 'error' and end_status_note = 'stg1_completed'
loop
select count(*) from transactional.test_instance where file_syspk = f.file_syspk into trx_record_count;
if trx_record_count > 0 then
update fw_core.fw_jobctl_file_runschedule set end_status_note ='generic_validation_failed', end_status='error', staging_type = 'stg1',end_time = now() where file_syspk = f.file_syspk;
update fw_core.fw_jobctl_file_sheet_runschedule set end_status_note = 'file_syspk exists', end_status='error', staging_type = 'stg1',end_time = now() where file_syspk = f.file_syspk;
end if;
update fw_core.fw_jobctl_file_runschedule set end_status_note ='generic_validation_completed', end_status='success', staging_type = 'stg1',end_time = now() where file_syspk = f.file_syspk;
update fw_core.fw_jobctl_file_sheet_runschedule set end_status_note = 'generic_validation_completed', end_status='success', staging_type = 'stg1',end_time = now() where file_syspk = f.file_syspk and file_sheet_mnemonic is not null;
end loop;
SELECT SUM (case WHEN end_status = 'success' THEN 1
ELSE 0 END) AS "Error Sum" FROM fw_core.fw_jobctl_file_sheet_runschedule where latest_runschedule_flag = '1' INTO success_sum;
if (success_sum >0 or success_sum is null) then
update fw_core.fw_jobctl_runschedule_jobstep set end_status_note = 'generic_validation_completed', end_status='success' , end_time= now() where job_id = w_job_id and step_id = w_step_id and latest_runschedule_flag = '1';
else
update fw_core.fw_jobctl_runschedule_jobstep set end_status_note = 'generic_validation_failed', end_status='error', end_time= now() where job_id = w_job_id and step_id = w_step_id and latest_runschedule_flag = '1';
return ;
end if;
--insert into fw_core.track_jobstep select *,'file_syspk' from fw_core.fw_jobctl_runschedule_jobstep;
--insert into fw_core.track_file select *,'file_syspk' from fw_core.fw_jobctl_file_runschedule;
--insert into fw_core.track_file_sheet select * ,'file_syspk' from fw_core.fw_jobctl_file_sheet_runschedule;
-- validate sheet names for a given file:
for f in select distinct file_syspk,file_mnemonic,file_sheet_mnemonic from fw_core.fw_jobctl_file_sheet_runschedule
where file_sheet_mnemonic is not null
and end_status != 'error' and end_status_note = 'generic_validation_completed' and latest_runschedule_flag = '1'
loop
select count(distinct sheet_name) from fw_core.test_file_sheet_format_master
where sheet_name not in
(select distinct sheet_name from fw_core.fw_jobctl_file_sheet_runschedule where file_mnemonic in (f.file_mnemonic) and file_sheet_mnemonic is not null and file_syspk=f.file_syspk)
and file_mnemonic in (f.file_mnemonic)
into missing_sheet_count;
if missing_sheet_count = 0 then
update fw_core.fw_jobctl_file_sheet_runschedule set end_status_note = 'generic_validation_completed', end_status='success', staging_type = 'stg1',end_time = now() where
file_syspk = f.file_syspk and file_sheet_mnemonic=f.file_sheet_mnemonic;
update fw_core.fw_jobctl_file_runschedule set end_status_note ='generic_validation_completed', end_status='success', staging_type = 'stg1',end_time = now() where file_syspk = f.file_syspk;
else
update fw_core.fw_jobctl_file_sheet_runschedule set end_status_note = 'stg1_sheet_missing', end_status='error', staging_type = 'stg1',end_time = now() where
file_syspk = f.file_syspk and file_sheet_mnemonic=f.file_sheet_mnemonic;
update fw_core.fw_jobctl_file_runschedule set end_status_note ='generic_validation_failed', end_status='error', staging_type = 'stg1',end_time = now() where file_syspk = f.file_syspk;
end if;
end loop;
SELECT SUM (case WHEN end_status = 'success' THEN 1
ELSE 0 END) AS "Error Sum" FROM fw_core.fw_jobctl_file_sheet_runschedule where latest_runschedule_flag = '1' INTO success_sum;
if ((success_sum > 0) or (success_sum is null)) then
update fw_core.fw_jobctl_runschedule_jobstep set end_status_note = 'generic_validation_completed', end_status='success', end_time=now() where job_id = w_job_id and step_id = w_step_id and latest_runschedule_flag = '1';
else
update fw_core.fw_jobctl_runschedule_jobstep set end_status_note = 'generic_validation_failed', end_status='error', end_time=now() where job_id = w_job_id and step_id = w_step_id and latest_runschedule_flag = '1';
return ;
end if;
insert into fw_core.track_jobstep select *,'missing_sheet' from fw_core.fw_jobctl_runschedule_jobstep;
insert into fw_core.track_file select *,'missing_sheet' from fw_core.fw_jobctl_file_runschedule;
insert into fw_core.track_file_sheet select * ,'missing_sheet' from fw_core.fw_jobctl_file_sheet_runschedule;
-- validate keywords for a given file:
for f in select distinct file_syspk,file_mnemonic,file_sheet_mnemonic
from fw_core.fw_jobctl_file_sheet_runschedule where file_sheet_mnemonic !=''
and end_status != 'error' and end_status_note = 'generic_validation_completed'
order by file_syspk
loop
select case
when file_sheet_mnemonic in ('FTDRY_SUM','FTHLG_SUM')then count(trim(column2))-2
when file_sheet_mnemonic in ('FTWET_SUM')then count(trim(column2))-1
else count(trim(column2)) end
from staging1.staging_generic_table where column2
in (select trim(f1_keyword) from transactional.source_config a
where a.file_mnemonic=f.file_mnemonic and a.file_sheet_mnemonic=f.file_sheet_mnemonic )
and file_mnemonic=f.file_mnemonic and file_sheet_mnemonic=f.file_sheet_mnemonic and file_syspk = f.file_syspk group by file_sheet_mnemonic into keyword_match_count;
if keyword_match_count != (select count(f1_keyword) from transactional.source_config a
where a.file_mnemonic=f.file_mnemonic and a.file_sheet_mnemonic=f.file_sheet_mnemonic)
then
update fw_core.fw_jobctl_file_sheet_runschedule set end_status_note = 'stg1_keyword_validation_failed', end_status='error', staging_type = 'stg1',end_time = now() where
file_syspk = f.file_syspk and file_sheet_mnemonic=f.file_sheet_mnemonic;
else
update fw_core.fw_jobctl_file_sheet_runschedule set end_status_note = 'generic_validation_completed', end_status='success', staging_type = 'stg1',end_time = now() where
file_syspk = f.file_syspk and file_sheet_mnemonic=f.file_sheet_mnemonic;
end if;
end loop;
-- update fw_jobctl_file_runschedule table.
--insert into fw_core.track_jobstep select *,'keyword_' from fw_core.fw_jobctl_runschedule_jobstep;
--insert into fw_core.track_file select *,'keyword_' from fw_core.fw_jobctl_file_runschedule;
--insert into fw_core.track_file_sheet select *,'keyword_' from fw_core.fw_jobctl_file_sheet_runschedule;
--, staging_type = 'stg1'
update fw_core.fw_jobctl_file_runschedule set end_status_note = 'generic_validation_completed', end_status='success', end_time=now() from fw_core.fw_jobctl_file_sheet_runschedule a
where a.latest_runschedule_flag = '1' and a.end_status = 'success' and a.end_status_note ='generic_validation_completed' and fw_core.fw_jobctl_file_runschedule.file_syspk = a.file_syspk;
update fw_core.fw_jobctl_file_runschedule set end_status_note = 'generic_validation_failed' , end_status='error',end_time=now() from fw_core.fw_jobctl_file_sheet_runschedule a
where a.latest_runschedule_flag = '1' and a.end_status = 'error' and a.end_status_note = 'stg1_keyword_validation_failed' and fw_core.fw_jobctl_file_runschedule.file_syspk = a.file_syspk;
update fw_core.fw_jobctl_file_sheet_runschedule set end_status='error',end_status_note = 'stg1_keyword_validation_failed',end_time=now() from fw_core.fw_jobctl_file_runschedule a
where file_sheet_mnemonic is not null and a.latest_runschedule_flag = '1' and a.end_status = 'error' and a.end_status_note = 'generic_validation_failed' and fw_core.fw_jobctl_file_sheet_runschedule.file_syspk = a.file_syspk;
SELECT SUM (case WHEN end_status = 'success' THEN 1
ELSE 0 END) AS "Error Sum" FROM fw_core.fw_jobctl_file_runschedule where latest_runschedule_flag = '1' INTO success_sum;
if (success_sum >0 or success_sum is null) then
update fw_core.fw_jobctl_runschedule_jobstep set end_status_note = 'generic_validation_completed', end_status='success', end_time=now() where job_id = w_job_id and step_id = w_step_id and latest_runschedule_flag = '1';
else
update fw_core.fw_jobctl_runschedule_jobstep set end_status_note = 'generic_validation_failed', end_status= 'error', end_time=now() where job_id = w_job_id and step_id = w_step_id and latest_runschedule_flag = '1';
end if;
--insert into fw_core.track_jobstep select *,'keyword_missing' from fw_core.fw_jobctl_runschedule_jobstep;
--insert into fw_core.track_file select *,'keyword_missing' from fw_core.fw_jobctl_file_runschedule;
--insert into fw_core.track_file_sheet select *,'keyword_missing' from fw_core.fw_jobctl_file_sheet_runschedule;
end;
$$ LANGUAGE plpgsql;