#!/bin/bash -e #Declare the Associative array declare -A credentials HOME_DIR=$HOME echo $HOME_DIR CONF_DIR=$HOME_DIR/conf #Server credentials file path input_file=$CONF_DIR/server_credentials.txt #Read the credentials from the server_credentials from conf folders #Remove Blank lines and comment starting with # while IFS="=" read -r key value; do credentials[$key]=$value done < <(sed -e '/^\s*$/ d' -e '/^#/ d' $input_file) # done < <(sed -e '/^\s*$/ d' -e '/^#/ d' $input_file) #CODS credentials host=${credentials[host]} #echo ${credentials[host]} user=${credentials[user]} #echo ${credentials[user]} pwd=${credentials[password]} #echo ${credentials[password]} port=${credentials[port]} dbname=${credentials[dbname1]} psql_base_arg="postgresql://$user:$pwd@$host/$dbname" echo $psql_base_arg echo "Initiating a job" date /home/compegence/dev/retail/retail_monthly_job_load.sh #while loop for each run_frequency while IFS="|" read -a outer do client_id=${outer[0]} function_id=${outer[1]} run_frequency=${outer[2]} echo "$run_frequency" #Update start_time for fw_jobctl_runschedule psql $psql_base_arg -c "update fw_core.fw_jobctl_runschedule set start_time=now(),begin_status='started' where client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' and end_status is null " #while loop for each job & step while IFS="|" read -a inner do job_name=${inner[4]} step_name+=(${inner[5]}) job_script=${inner[6]} job_function_name=${inner[7]} echo "$job_function_name" #remove first and last char - curly braces job_fun_param=${inner[8]:1:-1} data_from_date==${inner[10]} data_to_date=${inner[11]} job_scope=${inner[12]} job_scope_qualifier=${inner[13]} #echo $data_from_date,$data_to_date,$job_scope,$job_scope_qualifier param='' var1=' ' echo "$job_fun_param" echo "$job_script" #reading array column and spliting each var and appending $ & space IFS=',' read -r -a ADDR <<< "$job_fun_param" for i in "${ADDR[@]}"; do echo $i param=$param$i$var1 done echo $param if [ $job_script == "R" ] ; then echo " $job_function_name started" psql $psql_base_arg -c "update fw_core.fw_jobctl_runschedule_jobstep set jobstep_start_time=now(),begin_status='started',job_status='started' where job_function_name='$job_function_name' and client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' and latest_flag='true' and job_script='R'" #Rscript /home/compegence/R_code/$job_function_name $host $port $user $pwd $dbname $param $client_id $function_id $data_from_date $data_to_date $job_scope $job_scope_qualifier echo "$job_scope_qualifier" Rscript /home/compegence/R_code/$job_function_name $host $port $user $pwd $dbname $param $client_id $function_id "2017-01-01" "2020-12-31" $job_scope $job_scope_qualifier psql $psql_base_arg -c "update fw_core.fw_jobctl_runschedule_jobstep set jobstep_end_time=now(),end_status='completed',job_status='completed',latest_flag='false' where job_function_name='$job_function_name' and client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' and latest_flag='true' and job_script='R'" echo " $job_function_name Ended" elif [ $job_script == "SQL" ] ; then echo " $job_function_name started" psql $psql_base_arg -c "update fw_core.fw_jobctl_runschedule_jobstep set jobstep_start_time=now(),begin_status='started',job_status='started' where job_function_name='$job_function_name' and client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' and latest_flag='true' and job_script='SQL'" # psql $psql_base_arg -a -f /home/compegence/R_code/$job_function_name $host $port $user $pwd $dbname $param $client_id $function_id $data_from_date $data_to_date $job_scope $job_scope_qualifier psql $psql_base_arg -a -f $job_function_name $host $port $user $pwd $dbname $param $client_id $function_id $data_from_date $data_to_date $job_scope $job_scope_qualifier psql $psql_base_arg -c "update fw_core.fw_jobctl_runschedule_jobstep set jobstep_end_time=now(),end_status='completed',job_status='completed',latest_flag='false' where job_function_name='$job_function_name' and client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' and latest_flag='true' and job_script='SQL'" echo " $job_function_name Ended" elif [ $job_script == "JAVA" ] ; then echo " $job_function_name started" psql $psql_base_arg -c "update fw_core.fw_jobctl_runschedule_jobstep set jobstep_start_time=now(),begin_status='started',job_status='started' where job_function_name='$job_function_name' and client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' and latest_flag='true' and job_script='JAVA'" # java -jar /home/compegence/mmt_code/generic_table/target/ex1-1.0-SNAPSHOT-shaded.jar # java -jar $job_function_name psql $psql_base_arg -c "update fw_core.fw_jobctl_runschedule_jobstep set jobstep_end_time=now(),end_status='completed',job_status='completed',latest_flag='false' where job_function_name='$job_function_name' and client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' and latest_flag='true' and job_script='JAVA'" echo " $job_function_name Ended" fi done < <(psql $psql_base_arg -At -c"select client_id,function_id,run_frequency,run_schedule_id,job_name,step_name,job_script,job_function_name,job_fun_param,job_step_run_dependency_seuqence,data_from_date,data_to_date,job_scope,job_scope_qualifier from fw_core.fw_jobctl_runschedule_jobstep where latest_flag='true' and client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' order by run_schedule_id,job_step_run_dependency_seuqence; ") #Update end_time for fw_jobctl_runschedule psql $psql_base_arg -c "update fw_core.fw_jobctl_runschedule set end_time=now(),end_status='stg1_completed' where client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' and end_status is null " done < <(psql $psql_base_arg -At -c"select distinct client_id,function_id,run_frequency from fw_core.fw_jobctl_runschedule_jobstep where latest_flag='true' order by client_id,function_id,run_frequency ")