dheepa/mmt_sql
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

sql files

Browse Source
This commit is contained in:
dheepa
2021-07-02 08:40:37 +00:00
commit a0d3257be0
154 changed files with 49773 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
scripts/.run_onetime.sh.swm Executable file
View File

Binary file not shown.

BIN
scripts/.run_onetime.sh.swn Executable file
View File

Binary file not shown.

BIN
scripts/.run_onetime.sh.swo Executable file
View File

Binary file not shown.

BIN
scripts/.run_onetime.sh.swp Executable file
View File

Binary file not shown.

BIN
scripts/.run_onetime_v2.sh.swp Executable file
View File

Binary file not shown.

BIN
scripts/.run_wrapper_v2.sh.swp Executable file
View File

Binary file not shown.

50
scripts/check_duplicate_file.sh Executable file
View File

@@ -0,0 +1,50 @@
#!/bin/bash -e
HOME="/home/compegence/customer"
#Directories
homeDir=$HOME/MMT
configDir=$homeDir/conf
input_file=$configDir/etl-config.txt
declare -A credentials
while IFS="=" read -r key value; do
credentials[$key]=$value
done < <( sed -e '/^\s*$/ d' -e '/^#/ d' $input_file )
#for i in ${credentials[@]}; do echo ***$i; done
user=${credentials[user]}
password=${credentials[password]}
host=${credentials[host]}
db=${credentials[db]}
protocol=${credentials[protocol]}
server=${credentials[server]}
postgres_con="psql -A -t postgresql://$user:$password@$host/$db"
count=`$postgres_con -c "select count(*) from fw_core.fw_jobctl_file_runschedule"`
if [[ $count -eq 0 ]]
then
$postgres_con -q -c "update fw_core.fw_jobctl_runschedule_jobstep set end_status_note ='no_duplicate_file_found', end_status='success',end_time = now() where job_script_type = 'shell_script' and job_id = 1 and step_id= 1 and latest_runschedule_flag='1'"
else
while IFS= read -a filenames
do
file_name=$(echo $filenames | sed 's/[[:space:]]/\\ /g')
if [ $(echo "-f $homeDir/files/landing/$file_name") ];
then
echo "File: $file_name is already processed and is moved to Duplicate Folder"
move=$(echo "mv $homeDir/files/landing/$file_name $homeDir/files/duplicate_file/$file_name")
eval $move
#update ctl table to exit for duplicate files
$postgres_con -q -c "update fw_core.fw_jobctl_runschedule_jobstep set end_status_note ='duplicate_file_found', end_status='success',end_time = now() where job_script_type = 'shell_script' and job_id = 1 and step_id= 1 and latest_runschedule_flag='1'"
else
$postgres_con -q -c "update fw_core.fw_jobctl_runschedule_jobstep set end_status_note = 'no_duplicate_file_found', end_status='success',end_time = now() where job_script_type = 'shell_script' and job_id = 1 and step_id= 1 and latest_runschedule_flag='1'"
fi
done < <($postgres_con -c "select file_name from fw_core.fw_jobctl_file_runschedule where end_status_note='TRX_completed' and end_status ='success'")
fi
#$postgres_con -c "insert into fw_core.track_jobstep select *,'duplicate_check' from fw_core.fw_jobctl_runschedule_jobstep;"
#$postgres_con -c "insert into fw_core.track_file select *,'duplicate_check' from fw_core.fw_jobctl_file_runschedule;"
#$postgres_con -c "insert into fw_core.track_file_sheet select *,'duplicate_check' from fw_core.fw_jobctl_file_sheet_runschedule;"

61
scripts/file_move.sh Executable file
View File

@@ -0,0 +1,61 @@
#!/bin/bash -e
HOME="/home/compegence/customer"
#Directories
homeDir=$HOME/MMT
configDir=$homeDir/conf
input_file=$configDir/etl-config.txt
declare -A credentials
while IFS="=" read -r key value; do
credentials[$key]=$value
done < <( sed -e '/^\s*$/ d' -e '/^#/ d' $input_file )
user=${credentials[user]}
password=${credentials[password]}
host=${credentials[host]}
db=${credentials[db]}
protocol=${credentials[protocol]}
server=${credentials[server]}
postgres_con="psql -A -t postgresql://$user:$password@$host/$db"
filesyspks=`$postgres_con -q -c "select file_syspk from fw_core.fw_jobctl_file_runschedule where latest_runschedule_flag='1' /*and end_status_note='TRX_completed'*/;"`
for filesyspk in ${filesyspks}
do
IFS="|" read name endstatus endnote <<<`$postgres_con -c "select file_name,end_status,end_status_note from fw_core.fw_jobctl_file_runschedule where file_syspk=$filesyspk"`
filename=$(echo $name | sed 's/[[:space:]]/\\ /g')
if [ $endstatus == "success" -a $endnote == 'TRX_completed' ]
then
echo " moving file:$filename to processed folder"
move=$(echo "mv $homeDir/files/landing/$filename $homeDir/files/processed/$filename")
eval $move
else
echo "moving file: $filename to error_file folder "
move=$(echo "mv $homeDir/files/landing/$filename $homeDir/files/error_file/$filename")
eval $move
fi
$postgres_con -q -c "update fw_core.fw_jobctl_runschedule_jobstep set end_status_note ='file_movement_completed', end_status='success',end_time = now() where job_script_type = 'shell_script' and latest_runschedule_flag='1' and job_name= 'file_cleanup'"
echo -e ' '
done
#$postgres_con -c "insert into fw_core.track_jobstep select *,'move_check' from fw_core.fw_jobctl_runschedule_jobstep;"
#$postgres_con -c "insert into fw_core.track_file select *,'move_check' from fw_core.fw_jobctl_file_runschedule;"
#$postgres_con -c "insert into fw_core.track_file_sheet select *,'move_check' from fw_core.fw_jobctl_file_sheet_runschedule;"

93
scripts/job_load.sh Executable file
View File

@@ -0,0 +1,93 @@
#!/bin/bash -e
#!/bin/bash -e
HOME="/home/compegence/customer"
#Directories
homeDir=$HOME/MMT
configDir=$homeDir/conf
input_file=$configDir/etl-config.txt
declare -A credentials
while IFS="=" read -r key value; do
credentials[$key]=$value
done < <( sed -e '/^\s*$/ d' -e '/^#/ d' $input_file )
user=${credentials[user]}
password=${credentials[password]}
host=${credentials[host]}
db=${credentials[db]}
protocol=${credentials[protocol]}
server=${credentials[server]}
#let i=0
#set -A arr
#BASE_CODS_ARGUMENTS="--host=$cods_host --user=$cods_user --password=$cods_pwd --port=$3306"
#BASE_ODS_ARGUMENTS="--user=$user --password=$pwd --host=$host"
psql_base_arg="postgresql://$user:$password@$host/$db"
#psql_base_arg="postgresql://postgres:j3(jLBq}e@localhost/ffm_k2"
#echo $psql_base_arg
ifStart=`date '+%d'`
#echo "$ifStart"
#Check today is sunday. Scheduling weekly job
today="$(date +%a)"
sun="Mon"
psql $psql_base_arg -q -c "update fw_core.fw_jobctl_runschedule set latest_runschedule_flag='0' where latest_runschedule_flag='1';"
psql $psql_base_arg -q -c "update fw_core.fw_jobctl_runschedule_jobstep set latest_runschedule_flag='0' where latest_runschedule_flag='1';"
psql $psql_base_arg -q -c "update fw_core.fw_jobctl_file_runschedule set latest_runschedule_flag='0' where latest_runschedule_flag='1';"
psql $psql_base_arg -q -c "update fw_core.fw_jobctl_file_sheet_runschedule set latest_runschedule_flag='0' where latest_runschedule_flag='1';"
#inserting into fw_core.fw_jobctl_runschedule
chkrun=$(psql $psql_base_arg -t -c "select count(*) from fw_core.fw_jobctl_jobstep_master where client_id=21 and function_id=1 and run_frequency='daily' and active_flag=true;")
if [[ $chkrun -gt 0 ]]; then
psql $psql_base_arg -q -c "INSERT INTO fw_core.fw_jobctl_runschedule
(client_id, function_id, run_schedule_timestamp, run_frequency, start_time, begin_status, created_by, updated_by, create_timestamp, update_timestamp)
select client_id, function_id,now(),run_frequency,now(),'','Admin','Admin',now(),now() from fw_core.fw_jobctl_jobstep_master
where active_flag='TRUE' group by client_id, function_id, run_frequency limit 1;"
#check previous day job status in fw_jobctl_runschedule
chkrun2=$(psql $psql_base_arg -t -c "select trim(end_status) from fw_core.fw_jobctl_runschedule where run_schedule_id = (
select max(run_schedule_id) as max from fw_core.fw_jobctl_runschedule where latest_runschedule_flag='0' and client_id=21 and function_id=1);")
if [ "$chkrun2" != " error" ]; then
#Update fw_jobctl_runschedule
psql $psql_base_arg -q -c "update fw_core.fw_jobctl_runschedule set begin_status='started' where client_id=21 and function_id=1 and run_frequency='daily' and latest_runschedule_flag='1' "
else
psql $psql_base_arg -q -c "update fw_core.fw_jobctl_runschedule set begin_status='started',end_status='error',end_status_note = 'previous day job failed' where client_id=21 and function_id=1 and run_frequency='daily' and latest_runschedule_flag='1' "
echo "welcome2"
exit 1;
fi
else
echo " No active job for daily for client_id=21"
exit 1;
fi
# looping fw_jobctl_jobstep_master for active records
while IFS="|" read -a line
do
run_frequency=${line[2]}
job_scheduling_day="${line[1]}"
#echo "$run_frequency"
if [ $run_frequency == "daily" ] ; then
echo "Populating daily job into fw_jobctl_runschedule_jobstep"
psql $psql_base_arg -q -c"insert into fw_core.fw_jobctl_runschedule_jobstep(client_id, function_id,latest_runschedule_flag,run_schedule_id,run_schedule_timestamp, run_frequency, job_id, step_id, job_step_run_dependency_seuqence,
job_script_type,job_step_script_name,job_fun_param_array,job_name, step_name, job_scope, job_scope_qualifier_array, created_by,updated_by, create_timestamp, update_timestamp)
select a.client_id, a.function_id,b.latest_runschedule_flag, b.run_schedule_id,b.run_schedule_timestamp,a.run_frequency, a.job_id, a.step_id, a.job_step_run_dependency_seuqence,
a.job_script_type,a.job_step_script_name,a.job_fun_param_array,a.job_name,a.step_name,a.job_scope, a.job_scope_qualifier_array, 'Admin','Admin',now(),now()
from fw_core.fw_jobctl_jobstep_master a join fw_core.fw_jobctl_runschedule b
on a.client_id=b.client_id and a.function_id=b.function_id and a.run_frequency=b.run_frequency
where latest_runschedule_flag='1' and b.begin_status = 'started' and b.end_status is null and a.active_flag='true' and a.run_frequency='daily'
order by a.client_id,a.function_id,a.job_id,a.step_id,a.job_step_run_dependency_seuqence;"
fi
done < <(psql $psql_base_arg -At -c"select distinct client_id,function_id,run_frequency from fw_core.fw_jobctl_jobstep_master order by run_frequency ")

48
scripts/run_onetime_compile.sh Executable file
View File

@@ -0,0 +1,48 @@
#!/bin/bash -e
HOME="/home/compegence/customer"
#Directories
homeDir=$HOME/MMT
configDir=$homeDir/conf
input_file=$configDir/etl-config.txt
declare -A credentials
while IFS="=" read -r key value; do
credentials[$key]=$value
done < <( sed -e '/^\s*$/ d' -e '/^#/ d' $input_file )
#for i in ${credentials[@]}; do echo ***$i; done
user=${credentials[user]}
password=${credentials[password]}
host=${credentials[host]}
db=${credentials[db]}
protocol=${credentials[protocol]}
server=${credentials[server]}
mmt_con="psql -X -q -A -t postgresql://$user:$password@$host/$db?options=--search_path%3D"
declare -a archivepathvar=("$homeDir/sql/onetime/archive/schema")
declare -a archivepathsvar=("$homeDir/sql/onetime/archive/create" "$homeDir/sql/onetime/archive/function")
declare -a trxpathvar=("$homeDir/sql/onetime/createtables/common" "$homeDir/sql/onetime/createtables/ods" "$homeDir/sql/onetime/createtables/control" "$homeDir/sql/onetime/commonfunctions" "$homeDir/sql/onetime/configentries" "$homeDir/sql/onetime/dataloadfunctions/ODS_Load")
declare -a stgpathvar=("$homeDir/sql/onetime/createtables/staging" "$homeDir/sql/onetime/dataloadfunctions/budni" "$homeDir/sql/onetime/dataloadfunctions/Drawbar" "$homeDir/sql/onetime/dataloadfunctions/IHT" "$homeDir/sql/onetime/dataloadfunctions/field" "$homeDir/sql/onetime/dataloadfunctions/PTO" "$homeDir/sql/onetime/wrappers")
func_execute_conn(){
local conn="$1"
local -n param=$2
for path in "${param[@]}"
do
for files in "$path"/*
do
$conn -f $files
done
done
}
func_execute_conn "$mmt_con'archive'" archivepathvar
func_execute_conn "$mmt_con'staging2'" stgpathvar
func_execute_conn "$mmt_con'trx'" trxpathvar
func_execute_conn "$mmt_con'archive'" archivepathsvar

149
scripts/run_schedule.sh Executable file
View File

@@ -0,0 +1,149 @@
#!/bin/bash -e
HOME="/home/compegence/customer"
#Directories
homeDir=$HOME/MMT
configDir=$homeDir/conf
input_file=$configDir/etl-config.txt
declare -A credentials
while IFS="=" read -r key value; do
credentials[$key]=$value
done < <( sed -e '/^\s*$/ d' -e '/^#/ d' $input_file )
user=${credentials[user]}
password=${credentials[password]}
host=${credentials[host]}
db=${credentials[db]}
protocol=${credentials[protocol]}
server=${credentials[server]}
psql_base_arg="postgresql://$user:$password@$host/$db"
# load the record into the fw_jobctl_runschedule accounting for run frequency (hourly, daily, monthly, yearly) and insert
/home/compegence/customer/MMT/sql/scripts/job_load.sh
##***** Run frequency loop; Outer Loop
#while loop for each client_id,function_id,run_frequency from fw_jobctl_runschedule_jobstep
while IFS="|" read -a outer
do
client_id=${outer[0]}
function_id=${outer[1]}
run_frequency=${outer[2]}
#Update start_time for fw_jobctl_runschedule
#psql $psql_base_arg -q -c "update fw_core.fw_jobctl_runschedule set start_time=now(),begin_status='started' where latest_runschedule_flag='1' and client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' and end_status is null "
##** Job and Step Loop; innner loop;
#while loop for each job & step
##** Job and Step Loop; innner loop; based on the entries on the table fw_jobctl_runschedule_jobstep
job_status=0
prev_job_seq=''
file_count=0
while IFS="|" read -a inner
do
client_id=${inner[0]}
function_id=${inner[1]}
run_frequency=${inner[2]}
job_id=${inner[4]}
step_id=${inner[5]}
job_name=${inner[6]}
#step_name=(${inner[7]})
job_script_type=${inner[8]}
job_step_script_name=${inner[9]}
#remove first and last char - curly braces
job_fun_param_array=${inner[10]:1:-1}
job_step_run_dependency_seuqence=${inner[11]}
param=''
var1=' '
job_step_script_name_concat=$job_step_script_name'('$client_id,$function_id,$job_id,$step_id')'
#reading array column and spliting each var and appending $ & space
IFS=',' read -r -a ADDR <<< "$job_fun_param"
for i in "${ADDR[@]}"; do
param=$param$i$var1
done
##get file count
file_count=$(psql $psql_base_arg -t -c "select object_count from fw_core.fw_jobctl_runschedule where latest_runschedule_flag='1';")
echo "file-count is $file_count"
echo "Executing Job: $job_name by checking Previous End Status"
## Process java Program
if [[ $job_script_type == "java" ]]; then
psql $psql_base_arg -t -q -c "update fw_core.fw_jobctl_runschedule_jobstep set start_time=now(),begin_status='started' where job_step_script_name='$job_step_script_name' and client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' and latest_runschedule_flag='1' "
if [[ $prev_job_seq == '' ]]; then
java -jar $job_step_script_name
else
chkrun=$(psql $psql_base_arg -t -c "select TRIM(end_status) from fw_core.fw_jobctl_runschedule_jobstep where latest_runschedule_flag='1' and job_step_run_dependency_seuqence=$prev_job_seq and client_id=21 and function_id=1;")
if [[ "$chkrun" == " success" ]]; then
java -jar $job_step_script_name
else
exit 1;
fi
fi
## Process sql files
elif [[ $job_script_type == "sql" ]] ; then
psql $psql_base_arg -t -q -c "update fw_core.fw_jobctl_runschedule_jobstep set start_time=now(),begin_status='started' where job_step_script_name='$job_step_script_name' and client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' and latest_runschedule_flag='1' "
if [[ $prev_job_seq == '' ]]; then
psql $psql_base_arg -a -f $job_step_script_name $host $port $user $pwd $dbname $param $client_id $function_id $data_from_date $data_to_date $job_scope $job_scope_qualifier
else
chkrun=$(psql $psql_base_arg -t -c "select TRIM(end_status) from fw_core.fw_jobctl_runschedule_jobstep where latest_runschedule_flag='1' and job_step_run_dependency_seuqence=$prev_job_seq and client_id=21 and function_id=1;")
if [[ $file_count -gt 0 ]]; then
psql $psql_base_arg -a -f $job_step_script_name $host $port $user $pwd $dbname $param $client_id $function_id $data_from_date $data_to_date $job_scope $job_scope_qualifier
else
exit 1;
fi
fi
## Process shell_script Program
elif [[ $job_script_type == "shell_script" ]] ; then
psql $psql_base_arg -t -q -c "update fw_core.fw_jobctl_runschedule_jobstep set start_time=now(),begin_status='started' where job_step_script_name='$job_step_script_name' and client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' and latest_runschedule_flag='1' "
if [[ $prev_job_seq == '' ]]; then
$job_step_script_name
else
chkrun=$(psql $psql_base_arg -t -c "select TRIM(end_status) from fw_core.fw_jobctl_runschedule_jobstep where latest_runschedule_flag='1' and job_step_run_dependency_seuqence=$prev_job_seq and client_id=21 and function_id=1;")
if [[ $file_count -gt 0 ]]; then
$job_step_script_name
else
exit 1;
fi
fi
## Process sql fn Program
elif [[ $job_script_type == "sql_fn" ]] ; then
psql $psql_base_arg -t -q -c "update fw_core.fw_jobctl_runschedule_jobstep set start_time=now(),begin_status='started' where job_step_script_name='$job_step_script_name' and client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' and latest_runschedule_flag='1' "
if [[ $prev_job_seq == '' ]]; then
psql $psql_base_arg -c "select $job_step_script_name_concat"
else
chkrun=$(psql $psql_base_arg -t -c "select TRIM(end_status) from fw_core.fw_jobctl_runschedule_jobstep where latest_runschedule_flag='1' and job_step_run_dependency_seuqence=$prev_job_seq and client_id=21 and function_id=1;")
if [[ $file_count -gt 0 ]]; then
psql $psql_base_arg -t -c "select $job_step_script_name_concat"
else
exit 1;
fi
fi
fi
prev_job_seq=$job_step_run_dependency_seuqence
## End of While Loop
#2nd looping through fw_core.fw_jobctl_runschedule_jobstep for each client_id,function_id,run_frequency,run_schedule_id1 is completed
done < <(psql $psql_base_arg -At -c"select client_id,function_id,run_frequency,run_schedule_id,job_id,step_id,job_name,step_name,job_script_type,job_step_script_name,job_fun_param_array,job_step_run_dependency_seuqence from fw_core.fw_jobctl_runschedule_jobstep where latest_runschedule_flag='1' and client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' order by client_id,function_id,run_schedule_id,job_id,step_id,job_step_run_dependency_seuqence; ")
#Update end_time for fw_jobctl_runschedule
psql $psql_base_arg -q -c "update fw_core.fw_jobctl_runschedule set end_status='success',end_status_note='run_completed' ,latest_runschedule_flag='0',end_time=now() where client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' and latest_runschedule_flag='1' "
#1st loop for client_id,function_id,run_frequency
done < <(psql $psql_base_arg -At -c"select distinct client_id,function_id,run_frequency from fw_core.fw_jobctl_runschedule_jobstep where latest_runschedule_flag='1' order by client_id,function_id,run_frequency ")