168 lines
7.7 KiB
Bash
Executable File
168 lines
7.7 KiB
Bash
Executable File
#!/bin/bash -e
|
|
|
|
#/****************************************************************
|
|
#****** ForeWarn Licensing and / or usage Terms and Conditions *****
|
|
#ForeWarn. The NextGen Insights Solution Platform
|
|
|
|
#Copyright © 2021 COMPEGENCE. All Rights Reserved
|
|
|
|
#ForeWarn is a product of COMPEGENCE.
|
|
|
|
#To be used only with a valid license from COMPEGENCE
|
|
|
|
#www.compegence.com info@compegence.com
|
|
#****************************************************************/
|
|
|
|
#HOME="/home/compegence/customer"
|
|
#Directories
|
|
homeDir=$HOME/customer/MMT
|
|
configDir=$homeDir/conf
|
|
input_file=$configDir/etl-config.txt
|
|
declare -A credentials
|
|
while IFS="=" read -r key value; do
|
|
credentials[$key]=$value
|
|
done < <( sed -e '/^\s*$/ d' -e '/^#/ d' $input_file )
|
|
|
|
|
|
user=${credentials[user]}
|
|
password=${credentials[password]}
|
|
host=${credentials[host]}
|
|
db=${credentials[db]}
|
|
protocol=${credentials[protocol]}
|
|
server=${credentials[server]}
|
|
|
|
|
|
|
|
psql_base_arg="postgresql://$user:$password@$host/$db"
|
|
|
|
# load the record into the fw_jobctl_runschedule accounting for run frequency (hourly, daily, monthly, yearly) and insert
|
|
$homeDir/sql/scripts/job_load.sh
|
|
|
|
##***** Run frequency loop; Outer Loop
|
|
#while loop for each client_id,function_id,run_frequency from fw_jobctl_runschedule_jobstep
|
|
|
|
while IFS="|" read -a outer
|
|
do
|
|
client_id=${outer[0]}
|
|
function_id=${outer[1]}
|
|
run_frequency=${outer[2]}
|
|
|
|
#Update start_time for fw_jobctl_runschedule
|
|
#psql $psql_base_arg -q -c "update fw_core.fw_jobctl_runschedule set start_time=now(),begin_status='started' where latest_runschedule_flag='1' and client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' and end_status is null "
|
|
|
|
##** Job and Step Loop; innner loop;
|
|
#while loop for each job & step
|
|
##** Job and Step Loop; innner loop; based on the entries on the table fw_jobctl_runschedule_jobstep
|
|
|
|
job_status=0
|
|
prev_job_seq=''
|
|
file_count=0
|
|
while IFS="|" read -a inner
|
|
do
|
|
client_id=${inner[0]}
|
|
function_id=${inner[1]}
|
|
run_frequency=${inner[2]}
|
|
job_id=${inner[4]}
|
|
step_id=${inner[5]}
|
|
job_name=${inner[6]}
|
|
#step_name=(${inner[7]})
|
|
job_script_type=${inner[8]}
|
|
job_step_script_name=${inner[9]}
|
|
#remove first and last char - curly braces
|
|
job_fun_param_array=${inner[10]:1:-1}
|
|
job_step_run_dependency_seuqence=${inner[11]}
|
|
|
|
param=''
|
|
var1=' '
|
|
job_step_script_name_concat=$job_step_script_name'('$client_id,$function_id,$job_id,$step_id')'
|
|
|
|
#reading array column and spliting each var and appending $ & space
|
|
IFS=',' read -r -a ADDR <<< "$job_fun_param"
|
|
for i in "${ADDR[@]}"; do
|
|
param=$param$i$var1
|
|
done
|
|
|
|
|
|
|
|
##get file count
|
|
file_count=$(psql $psql_base_arg -t -c "select object_count from fw_core.fw_jobctl_runschedule where latest_runschedule_flag='1';")
|
|
echo "file-count is $file_count"
|
|
|
|
echo "Executing Job: $job_name by checking Previous End Status"
|
|
## Process java Program
|
|
if [[ $job_script_type == "java" ]]; then
|
|
psql $psql_base_arg -t -q -c "update fw_core.fw_jobctl_runschedule_jobstep set start_time=now(),begin_status='started' where job_step_script_name='$job_step_script_name' and client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' and latest_runschedule_flag='1' "
|
|
echo "--$prev_job_seq"
|
|
if [[ $prev_job_seq == '' ]]; then
|
|
echo "calling java jar"
|
|
java -jar $job_step_script_name
|
|
|
|
else
|
|
chkrun=$(psql $psql_base_arg -t -c "select TRIM(end_status) from fw_core.fw_jobctl_runschedule_jobstep where latest_runschedule_flag='1' and job_step_run_dependency_seuqence=$prev_job_seq and client_id=20 and function_id=1;")
|
|
echo "chk $chkrun"
|
|
if [[ "$chkrun" == " success" ]]; then
|
|
java -jar $job_step_script_name
|
|
else
|
|
exit 1;
|
|
fi
|
|
fi
|
|
|
|
## Process sql files
|
|
elif [[ $job_script_type == "sql" ]] ; then
|
|
psql $psql_base_arg -t -q -c "update fw_core.fw_jobctl_runschedule_jobstep set start_time=now(),begin_status='started' where job_step_script_name='$job_step_script_name' and client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' and latest_runschedule_flag='1' "
|
|
if [[ $prev_job_seq == '' ]]; then
|
|
psql $psql_base_arg -a -f $job_step_script_name $host $port $user $pwd $dbname $param $client_id $function_id $data_from_date $data_to_date $job_scope $job_scope_qualifier
|
|
else
|
|
chkrun=$(psql $psql_base_arg -t -c "select TRIM(end_status) from fw_core.fw_jobctl_runschedule_jobstep where latest_runschedule_flag='1' and job_step_run_dependency_seuqence=$prev_job_seq and client_id=20 and function_id=1;")
|
|
if [[ "$chkrun" == " success" && $file_count -gt 0 ]]; then
|
|
psql $psql_base_arg -a -f $job_step_script_name $host $port $user $pwd $dbname $param $client_id $function_id $data_from_date $data_to_date $job_scope $job_scope_qualifier
|
|
else
|
|
exit 1;
|
|
fi
|
|
fi
|
|
|
|
## Process shell_script Program
|
|
elif [[ $job_script_type == "shell_script" ]] ; then
|
|
psql $psql_base_arg -t -q -c "update fw_core.fw_jobctl_runschedule_jobstep set start_time=now(),begin_status='started' where job_step_script_name='$job_step_script_name' and client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' and latest_runschedule_flag='1' "
|
|
if [[ $prev_job_seq == '' ]]; then
|
|
$job_step_script_name
|
|
else
|
|
chkrun=$(psql $psql_base_arg -t -c "select TRIM(end_status) from fw_core.fw_jobctl_runschedule_jobstep where latest_runschedule_flag='1' and job_step_run_dependency_seuqence=$prev_job_seq and client_id=20 and function_id=1;")
|
|
|
|
if [[ "$chkrun" == " success" && $file_count -gt 0 ]]; then
|
|
$job_step_script_name
|
|
else
|
|
exit 1;
|
|
fi
|
|
fi
|
|
|
|
## Process sql fn Program
|
|
elif [[ $job_script_type == "sql_fn" ]] ; then
|
|
psql $psql_base_arg -t -q -c "update fw_core.fw_jobctl_runschedule_jobstep set start_time=now(),begin_status='started' where job_step_script_name='$job_step_script_name' and client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' and latest_runschedule_flag='1' "
|
|
if [[ $prev_job_seq == '' ]]; then
|
|
psql $psql_base_arg -c "select $job_step_script_name_concat"
|
|
else
|
|
|
|
chkrun=$(psql $psql_base_arg -t -c "select TRIM(end_status) from fw_core.fw_jobctl_runschedule_jobstep where latest_runschedule_flag='1' and job_step_run_dependency_seuqence=$prev_job_seq and client_id=20 and function_id=1;")
|
|
if [[ "$chkrun" == " success" && $file_count -gt 0 ]]; then
|
|
psql $psql_base_arg -t -c "select $job_step_script_name_concat"
|
|
else
|
|
exit 1;
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
prev_job_seq=$job_step_run_dependency_seuqence
|
|
## End of While Loop
|
|
#2nd looping through fw_core.fw_jobctl_runschedule_jobstep for each client_id,function_id,run_frequency,run_schedule_id1 is completed
|
|
done < <(psql $psql_base_arg -At -c"select client_id,function_id,run_frequency,run_schedule_id,job_id,step_id,job_name,step_name,job_script_type,job_step_script_name,job_fun_param_array,job_step_run_dependency_seuqence from fw_core.fw_jobctl_runschedule_jobstep where latest_runschedule_flag='1' and client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' order by client_id,function_id,run_schedule_id,job_id,step_id,job_step_run_dependency_seuqence; ")
|
|
|
|
|
|
#Update end_time for fw_jobctl_runschedule
|
|
psql $psql_base_arg -q -c "update fw_core.fw_jobctl_runschedule set end_status='success',end_status_note='run_completed' ,latest_runschedule_flag='0',end_time=now() where client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' and latest_runschedule_flag='1' "
|
|
|
|
#1st loop for client_id,function_id,run_frequency
|
|
done < <(psql $psql_base_arg -At -c"select distinct client_id,function_id,run_frequency from fw_core.fw_jobctl_runschedule_jobstep where latest_runschedule_flag='1' order by client_id,function_id,run_frequency ")
|
|
|
|
|