dheepa/mmt_sql
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

mmt sql

Browse Source
This commit is contained in:
dheepa
2021-10-28 09:12:13 +00:00
parent f3cf5e1d2d
commit 8402c9b738
74 changed files with 3883 additions and 908 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
scripts/check_duplicate_file.sh
View File

@@ -1,8 +1,24 @@
#!/bin/bash -e
HOME="/home/compegence/customer"
#/****************************************************************
#****** ForeWarn Licensing and / or usage Terms and Conditions *****
#ForeWarn. The NextGen Insights Solution Platform
#Copyright © 2021 COMPEGENCE. All Rights Reserved
#ForeWarn is a product of COMPEGENCE.
#To be used only with a valid license from COMPEGENCE
#www.compegence.com info@compegence.com
#****************************************************************/
#HOME="/home/compegence/customer"
#Directories
homeDir=$HOME/MMT
fileDir="/data/customer/MMT"
homeDir=$HOME/customer/MMT
configDir=$homeDir/conf
input_file=$configDir/etl-config.txt
declare -A credentials
@@ -22,31 +38,33 @@ postgres_con="psql -A -t postgresql://$user:$password@$host/$db"
$postgres_con -q -c "update fw_core.fw_jobctl_runschedule_jobstep set end_status_note ='Initial', end_status='success',end_time = now() where job_script_type = 'shell_script' and job_id = 1 and step_id= 1 and latest_runschedule_flag='1'"
count=`$postgres_con -c "select count(*) from fw_core.fw_jobctl_file_runschedule"`
if [[ $count -eq 0 ]]
then
$postgres_con -q -c "update fw_core.fw_jobctl_runschedule_jobstep set end_status_note ='no_duplicate_file_found', end_status='success',end_time = now() where job_script_type = 'shell_script' and job_id = 1 and step_id= 1 and latest_runschedule_flag='1'"
$postgres_con -q -c "update fw_core.fw_jobctl_runschedule_jobstep set end_status_note ='file_found_to_process', end_status='success',end_time = now() where job_script_type = 'shell_script' and job_id = 1 and step_id= 1 and latest_runschedule_flag='1'"
else
while IFS= read -a filenames
do
file_name=$(echo $filenames | sed 's/[[:space:]]/\\ /g')
if [[ -f $homeDir/files/landing/$filenames ]];
if [[ -f $fileDir/files/landing/$filenames ]];
then
echo "File: $file_name is already processed and is moved to Duplicate Folder"
move=$(echo "mv $homeDir/files/landing/$file_name $homeDir/files/duplicate_file/$file_name")
move=$(echo "mv $fileDir/files/landing/$file_name $fileDir/files/duplicate_file/$file_name")
eval $move
#update ctl table to exit for duplicate files
$postgres_con -q -c "update fw_core.fw_jobctl_runschedule_jobstep set end_status_note ='duplicate_file_found', end_status='success',end_time = now() where job_script_type = 'shell_script' and job_id = 1 and step_id= 1 and latest_runschedule_flag='1'"
else
$postgres_con -q -c "update fw_core.fw_jobctl_runschedule_jobstep set end_status_note = 'no_duplicate_file_found', end_status='success',end_time = now() where job_script_type = 'shell_script' and job_id = 1 and step_id= 1 and latest_runschedule_flag='1'"
fi
done < <($postgres_con -c "select file_name from fw_core.fw_jobctl_file_runschedule where end_status_note='TRX_completed' and end_status ='success'")
filecount=find $fileDir/files/landing/ -type f -name "*.xlsx" 2> /dev/null |wc -l 1> /dev/null
if [[ $filecount -eq 0 ]]
then
$postgres_con -q -c "update fw_core.fw_jobctl_runschedule_jobstep set end_status_note = 'No files in Landing folder', end_status='success',end_time = now() where job_script_type = 'shell_script' and job_id = 1 and step_id= 1 and latest_runschedule_flag='1'"
else
$postgres_con -q -c "update fw_core.fw_jobctl_runschedule_jobstep set end_status_note = 'file_found_to_process', end_status='success',end_time = now() where job_script_type = 'shell_script' and job_id = 1 and step_id= 1 and latest_runschedule_flag='1'"
fi
fi
done < <($postgres_con -c "select file_name from fw_core.fw_jobctl_file_runschedule where end_status_note='ODS_completed' and end_status ='success'")
fi
#$postgres_con -c "insert into fw_core.track_jobstep select *,'duplicate_check' from fw_core.fw_jobctl_runschedule_jobstep;"
#$postgres_con -c "insert into fw_core.track_file select *,'duplicate_check' from fw_core.fw_jobctl_file_runschedule;"
#$postgres_con -c "insert into fw_core.track_file_sheet select *,'duplicate_check' from fw_core.fw_jobctl_file_sheet_runschedule;"