dheepa/mmt_sql
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

mmt sql

Browse Source
This commit is contained in:
dheepa
2021-10-28 09:12:13 +00:00
parent f3cf5e1d2d
commit 8402c9b738
74 changed files with 3883 additions and 908 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
scripts/.file_move.sh.swp Executable file
View File

Binary file not shown.

36
scripts/Output.html Executable file
View File

@@ -0,0 +1,36 @@
<h4>**Latest Files Update Status**<h4>
<table border="1">
<tr>
<th align="center">run_schedule_id</th>
<th align="center">file_syspk</th>
<th align="center">file_mnemonic</th>
<th align="center">file_name</th>
<th align="center">status</th>
<th align="center">load_time</th>
</tr>
<tr valign="top">
<td align="right">20</td>
<td align="right">245</td>
<td align="left">BUDNI</td>
<td align="left">20181002_BUDNI_MF 241 Planetary Plus V1.xlsx</td>
<td align="left">success</td>
<td align="left">2021-08-27 06:32:58.220188</td>
</tr>
</table>
<p>(1 row)<br />
</p>
<h4>**Error Files*****<h4>
<table border="1">
<tr>
<th align="center">run_schedule_id</th>
<th align="center">file_syspk</th>
<th align="center">file_mnemonic</th>
<th align="center">file_name</th>
<th align="center">sheet_id</th>
<th align="center">sheet_name</th>
<th align="center">end_status</th>
<th align="center">load_time</th>
</tr>
</table>
<p>(0 rows)<br />
</p>

57
scripts/Output.txt Executable file
View File

@@ -0,0 +1,57 @@
--asdfghjkl
Content-Type: text/plain; charset=utf-8
Alerts for the job run status
--asdfghjkl
Content-Type: text/html; charset=utf-8
<!DOCTYPE html>
<body>
<table border="0">
<tr valign="center">
<td align="center"><h4>Mahindra TDMS - Latest Flles Update Status</h4></td>
</tr>
</table>
<table border="1">
<tr>
<th align="center">run_schedule_id</th>
<th align="center">file_syspk</th>
<th align="center">file_mnemonic</th>
<th align="center">file_name</th>
<th align="center">status</th>
<th align="center">load_time</th>
</tr>
<tr valign="top">
<td align="right">70</td>
<td align="right">261</td>
<td align="left">FTDRY</td>
<td align="left">20210725_FTDRY_575 DI XP_45 HP_VS_SONALIKA_VS_POWERTRAC EURO 45 PLUS_ROTAVATOR_SHIVPURI.xlsx</td>
<td align="left">success</td>
<td align="left">2021-10-28 08:38:25.586439</td>
</tr>
</table>
<p>(1 row)<br />
</p>
<table border="0">
<tr valign="center">
<td align="center"><h4>Error Files</h4></td>
</tr>
</table>
<table border="1">
<tr>
<th align="center">run_schedule_id</th>
<th align="center">file_syspk</th>
<th align="center">file_mnemonic</th>
<th align="center">file_name</th>
<th align="center">sheet_id</th>
<th align="center">sheet_name</th>
<th align="center">end_status</th>
<th align="center">load_time</th>
</tr>
</table>
<p>(0 rows)<br />
</p>
</body>
</html>
--asdfghjkl

46
scripts/check_duplicate_file.sh
View File

@@ -1,8 +1,24 @@
#!/bin/bash -e
HOME="/home/compegence/customer"
#/****************************************************************
#****** ForeWarn Licensing and / or usage Terms and Conditions *****
#ForeWarn. The NextGen Insights Solution Platform
#Copyright © 2021 COMPEGENCE. All Rights Reserved
#ForeWarn is a product of COMPEGENCE.
#To be used only with a valid license from COMPEGENCE
#www.compegence.com info@compegence.com
#****************************************************************/
#HOME="/home/compegence/customer"
#Directories
homeDir=$HOME/MMT
fileDir="/data/customer/MMT"
homeDir=$HOME/customer/MMT
configDir=$homeDir/conf
input_file=$configDir/etl-config.txt
declare -A credentials
@@ -22,31 +38,33 @@ postgres_con="psql -A -t postgresql://$user:$password@$host/$db"
$postgres_con -q -c "update fw_core.fw_jobctl_runschedule_jobstep set end_status_note ='Initial', end_status='success',end_time = now() where job_script_type = 'shell_script' and job_id = 1 and step_id= 1 and latest_runschedule_flag='1'"
count=`$postgres_con -c "select count(*) from fw_core.fw_jobctl_file_runschedule"`
if [[ $count -eq 0 ]]
then
$postgres_con -q -c "update fw_core.fw_jobctl_runschedule_jobstep set end_status_note ='no_duplicate_file_found', end_status='success',end_time = now() where job_script_type = 'shell_script' and job_id = 1 and step_id= 1 and latest_runschedule_flag='1'"
$postgres_con -q -c "update fw_core.fw_jobctl_runschedule_jobstep set end_status_note ='file_found_to_process', end_status='success',end_time = now() where job_script_type = 'shell_script' and job_id = 1 and step_id= 1 and latest_runschedule_flag='1'"
else
while IFS= read -a filenames
do
file_name=$(echo $filenames | sed 's/[[:space:]]/\\ /g')
if [[ -f $homeDir/files/landing/$filenames ]];
if [[ -f $fileDir/files/landing/$filenames ]];
then
echo "File: $file_name is already processed and is moved to Duplicate Folder"
move=$(echo "mv $homeDir/files/landing/$file_name $homeDir/files/duplicate_file/$file_name")
move=$(echo "mv $fileDir/files/landing/$file_name $fileDir/files/duplicate_file/$file_name")
eval $move
#update ctl table to exit for duplicate files
$postgres_con -q -c "update fw_core.fw_jobctl_runschedule_jobstep set end_status_note ='duplicate_file_found', end_status='success',end_time = now() where job_script_type = 'shell_script' and job_id = 1 and step_id= 1 and latest_runschedule_flag='1'"
else
$postgres_con -q -c "update fw_core.fw_jobctl_runschedule_jobstep set end_status_note = 'no_duplicate_file_found', end_status='success',end_time = now() where job_script_type = 'shell_script' and job_id = 1 and step_id= 1 and latest_runschedule_flag='1'"
fi
done < <($postgres_con -c "select file_name from fw_core.fw_jobctl_file_runschedule where end_status_note='TRX_completed' and end_status ='success'")
filecount=find $fileDir/files/landing/ -type f -name "*.xlsx" 2> /dev/null |wc -l 1> /dev/null
if [[ $filecount -eq 0 ]]
then
$postgres_con -q -c "update fw_core.fw_jobctl_runschedule_jobstep set end_status_note = 'No files in Landing folder', end_status='success',end_time = now() where job_script_type = 'shell_script' and job_id = 1 and step_id= 1 and latest_runschedule_flag='1'"
else
$postgres_con -q -c "update fw_core.fw_jobctl_runschedule_jobstep set end_status_note = 'file_found_to_process', end_status='success',end_time = now() where job_script_type = 'shell_script' and job_id = 1 and step_id= 1 and latest_runschedule_flag='1'"
fi
fi
done < <($postgres_con -c "select file_name from fw_core.fw_jobctl_file_runschedule where end_status_note='ODS_completed' and end_status ='success'")
fi
#$postgres_con -c "insert into fw_core.track_jobstep select *,'duplicate_check' from fw_core.fw_jobctl_runschedule_jobstep;"
#$postgres_con -c "insert into fw_core.track_file select *,'duplicate_check' from fw_core.fw_jobctl_file_runschedule;"
#$postgres_con -c "insert into fw_core.track_file_sheet select *,'duplicate_check' from fw_core.fw_jobctl_file_sheet_runschedule;"

1
scripts/con-pwd.sh Executable file
View File

@@ -0,0 +1 @@
psql -h localhost -d mmt_deploy -U postgres -p 2345

1
scripts/con.sh Executable file
View File

@@ -0,0 +1 @@
psql -X -A -t postgresql://postgres:'j3(jLBq}e'@localhost/MMT_PROD?options=--search_path%3Dstaging2

2
scripts/con_admin.sh Executable file
View File

@@ -0,0 +1,2 @@
psql -X -A -t postgresql://fw_admin:'$HjNxDV3g=bJ=Tw9'@localhost/forewarn_tdm?options=--search_path%3Dstaging2

28
scripts/con_check.sh Executable file
View File

@@ -0,0 +1,28 @@
#HOME="/home/compegence/customer"
#Directories
fileDir="/data/customer/MMT"
homeDir=$HOME/customer/MMT
configDir=$homeDir/conf
input_file=$configDir/etl-config.txt
declare -A credentials
while IFS="=" read -r key value; do
credentials[$key]=$value
done < <( sed -e '/^\s*$/ d' -e '/^#/ d' $input_file )
#for i in ${credentials[@]}; do echo ***$i; done
user=${credentials[user]}
password=${credentials[password]}
host=${credentials[host]}
db=${credentials[db]}
protocol=${credentials[protocol]}
server=${credentials[server]}
postgres_con="psql -A -t postgresql://$user:$password@$host/$db"
count=`$postgres_con -c "select count(*) from staging1.temtable"`
echo $count

60
scripts/correlation.sh Executable file
View File

@@ -0,0 +1,60 @@
#!/bin/bash -e
#/****************************************************************
#****** ForeWarn Licensing and / or usage Terms and Conditions *****
#ForeWarn. The NextGen Insights Solution Platform
#Copyright ¿ 2021 COMPEGENCE. All Rights Reserved
#ForeWarn is a product of COMPEGENCE.
#To be used only with a valid license from COMPEGENCE
#www.compegence.com info@compegence.com
#****************************************************************/
HOME="/home/compegence/customer"
#Directories
homeDir=$HOME/MMT
configDir=$homeDir/conf
input_file=$configDir/etl-config.txt
declare -A credentials
while IFS="=" read -r key value; do
credentials[$key]=$value
done < <( sed -e '/^\s*$/ d' -e '/^#/ d' $input_file )
user=${credentials[user]}
password=${credentials[password]}
host=${credentials[host]}
db=${credentials[db]}
protocol=${credentials[protocol]}
server=${credentials[server]}
psql_base_arg="postgresql://$user:$password@$host/$db"
#update fw_jobctl_file_landing
##get file mnemonic count
file_mnemonic_count_drawbar=$(psql $psql_base_arg -t -c "select count(*) from fw_core.fw_jobctl_file_runschedule where latest_runschedule_flag=true and file_mnemonic like 'DB%' ;")
echo "file-mnemonic-count is $file_mnemonic_count_drawbar"
if [[ $file_mnemonic_count_drawbar -gt 0 ]]; then
psql $psql_base_arg -t -c "select fw_ods.drawbar_correlation_upload()"
# else
# exit 1;
fi
file_mnemonic_count_pto=$(psql $psql_base_arg -t -c "select count(*) from fw_core.fw_jobctl_file_runschedule where latest_runschedule_flag=true and file_mnemonic like 'PTO%';")
echo "file-mnemonic-count_pto is $file_mnemonic_count_pto"
if [[ $file_mnemonic_count_pto -gt 0 ]]; then
psql $psql_base_arg -t -c "select fw_ods.pto_correlation_upload()"
# else
# exit 1;
fi

1
scripts/crontab Executable file
View File

@@ -0,0 +1 @@
25 13 * * * /home/tdmsadmin/customer/MMT/sql/scripts/run_schedule.sh >>/data/cumstomer/MMT/logs/test_data_MMT`date+\%Y\%m\%d\%H\%M\%S`.log 2>&1

43
scripts/fileLoadMail.sh Executable file
View File

@@ -0,0 +1,43 @@
#!/bin/bash -e
#Declare the Associative array
homeDir=$HOME/customer/MMT
configDir=$homeDir/conf
input_file=$configDir/etl-config.txt
declare -A credentials
while IFS="=" read -r key value; do
credentials[$key]=$value
done < <( sed -e '/^\s*$/ d' -e '/^#/ d' $input_file )
user=${credentials[user]}
password=${credentials[password]}
host=${credentials[host]}
db=${credentials[db]}
protocol=${credentials[protocol]}
server=${credentials[server]}
postgres_con="psql -A -t postgresql://$user:$password@$host/$db"
echo $postgres_con
(
echo "*****************************"
echo "**Latest File Update Status**"
echo "*****************************"
echo " "
psql $postgres_con -c "select run_schedule_id, file_syspk, file_mnemonic, file_name, end_status as status, end_time from fw_core.fw_jobctl_file_runschedule
where run_schedule_id in (select max(run_schedule_id) from fw_core.fw_jobctl_file_runschedule) order by run_schedule_id, file_syspk, end_status desc;"
echo " "
echo "**********************************"
echo "**Error Files********************"
echo "**********************************"
echo " "
psql $postgres_con -c "select A.run_schedule_id, A.file_syspk, A.file_mnemonic, A.file_name, B.sheet_id, B.sheet_name, A.end_status, A.end_time from fw_core.fw_jobctl_file_runschedule a, fw_core.fw_jobctl_file_sheet_runschedule b
where a.run_schedule_id = (select max(run_schedule_id) from fw_core.fw_jobctl_file_runschedule) and a.file_syspk = b.file_syspk and (b.file_sheet_mnemonic != '' or b.file_sheet_mnemonic is not null)
and b.end_status='error' order by A.run_schedule_id, b.file_syspk, b.sheet_id desc limit 10"
) | mail -s 'FIles Load Status' muin.shariff@compegence.com abdur.rahman@compegence.com dheepa.k@compegence.com jagadish.puvvada@compegence.com -a "From: noreply@forewarn.com"

78
scripts/fileLoadMailhtml.sh Executable file
View File

@@ -0,0 +1,78 @@
#!/bin/bash -e
#HOME="/home/compegence/customer"
#Directories
homeDir=$HOME/customer/MMT
configDir=$homeDir/conf
input_file=$configDir/etl-config.txt
declare -A credentials
while IFS="=" read -r key value; do
credentials[$key]=$value
done < <( sed -e '/^\s*$/ d' -e '/^#/ d' $input_file )
user=${credentials[user]}
password=${credentials[password]}
host=${credentials[host]}
db=${credentials[db]}
protocol=${credentials[protocol]}
server=${credentials[server]}
psql_base_arg="postgresql://$user:$password@$host/$db"
#dbname="mmt_tdm"
#port=5432
Get_Email_Ids () {
#client_id=$1
echo $client_id
email_ids=`psql $psql_base_arg -t -c "select runschedule_status_email from fw_core.fw_m_client where client_id=$client_id"`
#echo $email_ids
#mail -a "Content-type: text/html" -s "Mail configuration Completed" $email_ids -a "From: noreply@forewarn.com" < Output.html
}
#echo $psql_base_arg
Get_FileUpload_Status (){
#echo -e "MIME-Version: 1.0\nContent-Type: multipart/plain" >Output.txt
echo -e "--asdfghjkl\nContent-Type: text/plain; charset=utf-8\n\nAlerts for the job run status\n\n--asdfghjkl\nContent-Type: text/html; charset=utf-8\n\n<!DOCTYPE html>\n<body>\n<table border=\"0\">\n\t<tr valign=\"center\">\n\t\t<td align=\"center\"><h4>Mahindra TDMS - Latest Flles Update Status</h4></td>\n\t</tr>\n</table>" > Output.txt
#echo "<h4>Mahindra TDMS - Latest Files Update Status<h4>" >> Output.txt
#echo " " >>Output.txt
psql $psql_base_arg -H -c "select run_schedule_id, file_syspk, file_mnemonic, file_name, end_status as status, end_time load_time from fw_core.fw_jobctl_file_runschedule
where run_schedule_id in (select max(run_schedule_id) from fw_core.fw_jobctl_file_runschedule) order by run_schedule_id, file_syspk, end_status desc;" >> Output.txt
echo -e "<table border=\"0\">\n\t<tr valign=\"center\">\n\t\t<td align=\"center\"><h4>Error Files</h4></td>\n\t</tr>\n</table>" >> Output.txt
psql $psql_base_arg -H -c "select A.run_schedule_id, A.file_syspk, A.file_mnemonic, A.file_name, B.sheet_id, B.sheet_name, A.end_status, A.end_time load_time from fw_core.fw_jobctl_file_runschedule a, fw_core.fw_jobctl_file_sheet_runschedule b
where a.run_schedule_id = (select max(run_schedule_id) from fw_core.fw_jobctl_file_runschedule) and a.file_syspk = b.file_syspk and (b.file_sheet_mnemonic != '' or b.file_sheet_mnemonic is not null) and b.end_status='error' order by A.run_schedule_id, b.file_syspk, b.sheet_id desc" >> Output.txt
echo -e "</body>\n</html>\n--asdfghjkl" >> Output.txt
}
Send_Mail () {
Get_Email_Ids
echo $email_ids
mail -a "Content-Type: multipart/alternative; boundary=asdfghjkl" -s "FIle Load Status" $email_ids -a "From: noreply@forewarn.com" < Output.txt
}
#mail -a "Content-type: text/html" -s "Mail configuration Completed" muin.shariff@compegence.com nagaraj.kulkarni@compegence.com dheepa.k@compegence.com abdur.rahman@compegence.com jagadish.puvvada@compegence.com -a "From: noreply@forewarn.com" < Output.html
if [ -z "$1" ]
then
client_id=20
Get_FileUpload_Status
Send_Mail;
else
Get_FileUpload_Status
Send_Mail;
fi
psql $psql_base_arg -q -c "update fw_core.fw_jobctl_runschedule_jobstep set end_status_note ='email_completed', end_status='success',end_time = now() where job_script_type = 'shell_script' and latest_runschedule_flag='1' and job_name= 'sending mail'"

72
scripts/fileLoadMailhtml_20210928.sh Executable file
View File

@@ -0,0 +1,72 @@
#!/bin/bash -e
#HOME="/home/compegence/customer"
#Directories
homeDir=$HOME/customer/MMT
configDir=$homeDir/conf
input_file=$configDir/etl-config.txt
declare -A credentials
while IFS="=" read -r key value; do
credentials[$key]=$value
done < <( sed -e '/^\s*$/ d' -e '/^#/ d' $input_file )
user=${credentials[user]}
password=${credentials[password]}
host=${credentials[host]}
db=${credentials[db]}
protocol=${credentials[protocol]}
server=${credentials[server]}
psql_base_arg="postgresql://$user:$password@$host/$db"
Get_Email_Ids () {
#client_id=$1
# echo $client_id
email_ids=`psql $psql_base_arg -t -c "select runschedule_status_email from fw_core.fw_m_client where client_id=$client_id"`
#echo $email_ids
#mail -a "Content-type: text/html" -s "Mail configuration Completed" $email_ids -a "From: noreply@forewarn.com" < Output.html
}
#echo $psql_base_arg
Get_FileUpload_Status (){
#echo -e "MIME-Version: 1.0\nContent-Type: multipart/plain" >Output.txt
echo "<h4>**Latest Files Update Status**<h4>" > Output.html
#echo " " >>Output.txt
psql $psql_base_arg -H -c "select run_schedule_id, file_syspk, file_mnemonic, file_name, end_status as status, end_time load_time from fw_core.fw_jobctl_file_runschedule
where run_schedule_id in (select max(run_schedule_id) from fw_core.fw_jobctl_file_runschedule) order by run_schedule_id, file_syspk, end_status desc;" >> Output.html
echo "<h4>**Error Files*****<h4>" >> Output.html
psql $psql_base_arg -H -c "select A.run_schedule_id, A.file_syspk, A.file_mnemonic, A.file_name, B.sheet_id, B.sheet_name, A.end_status, A.end_time load_time from fw_core.fw_jobctl_file_runschedule a, fw_core.fw_jobctl_file_sheet_runschedule b
where a.run_schedule_id = (select max(run_schedule_id) from fw_core.fw_jobctl_file_runschedule) and a.file_syspk = b.file_syspk and (b.file_sheet_mnemonic != '' or b.file_sheet_mnemonic is not null) and b.end_status='error' order by A.run_schedule_id, b.file_syspk, b.sheet_id desc" >> Output.html
}
Send_Mail () {
Get_Email_Ids
echo $email_ids
mail -a "Content-type: text/html" -s "Mail configuration Completed" $email_ids -a "From: noreply@forewarn.com" < Output.html
}
#mail -a "Content-type: text/html" -s "Mail configuration Completed" muin.shariff@compegence.com nagaraj.kulkarni@compegence.com dheepa.k@compegence.com abdur.rahman@compegence.com jagadish.puvvada@compegence.com -a "From: noreply@forewarn.com" < Output.html
if [ -z "$1" ]
then
echo "No argument supplied"
echo "usage --> `basename ${0}` client_id"
echo "Ex: `basename ${0}` 20"
exit 1;
else
client_id=$1
Get_FileUpload_Status
Send_Mail;
fi

25
scripts/file_move.sh
View File

@@ -1,7 +1,22 @@
#!/bin/bash -e
HOME="/home/compegence/customer"
#/****************************************************************
#****** ForeWarn Licensing and / or usage Terms and Conditions *****
#ForeWarn. The NextGen Insights Solution Platform
#Copyright © 2021 COMPEGENCE. All Rights Reserved
#ForeWarn is a product of COMPEGENCE.
#To be used only with a valid license from COMPEGENCE
#www.compegence.com info@compegence.com
#****************************************************************/
#HOME="/home/compegence/customer"
#Directories
homeDir=$HOME/MMT
fileDir="/data/customer/MMT"
homeDir=$HOME/customer/MMT
configDir=$homeDir/conf
input_file=$configDir/etl-config.txt
declare -A credentials
@@ -25,15 +40,15 @@ for filesyspk in ${filesyspks}
do
IFS="|" read name endstatus endnote <<<`$postgres_con -c "select file_name,end_status,end_status_note from fw_core.fw_jobctl_file_runschedule where file_syspk=$filesyspk"`
filename=$(echo $name | sed 's/[[:space:]]/\\ /g')
if [ $endstatus == "success" -a $endnote == 'TRX_completed' ]
if [ $endstatus == "success" -a $endnote == 'ODS_completed' ]
then
echo " moving file:$filename to processed folder"
move=$(echo "mv $homeDir/files/landing/$filename $homeDir/files/processed/$filename")
move=$(echo "mv $fileDir/files/landing/$filename $fileDir/files/processed/$filename")
eval $move
else
echo "moving file: $filename to error_file folder "
move=$(echo "mv $homeDir/files/landing/$filename $homeDir/files/error_file/$filename")
move=$(echo "mv $fileDir/files/landing/$filename $fileDir/files/error_file/$filename")
eval $move
fi

28
scripts/job_load.sh
View File

@@ -1,9 +1,21 @@
#!/bin/bash -e
#/****************************************************************
#****** ForeWarn Licensing and / or usage Terms and Conditions *****
#ForeWarn. The NextGen Insights Solution Platform
#Copyright © 2021 COMPEGENCE. All Rights Reserved
#ForeWarn is a product of COMPEGENCE.
#To be used only with a valid license from COMPEGENCE
#www.compegence.com info@compegence.com
#****************************************************************/
#!/bin/bash -e
HOME="/home/compegence/customer"
#HOME="/home/compegence/customer"
#Directories
homeDir=$HOME/MMT
homeDir=$HOME/customer/MMT
configDir=$homeDir/conf
input_file=$configDir/etl-config.txt
declare -A credentials
@@ -43,7 +55,7 @@ sun="Mon"
psql $psql_base_arg -q -c "update fw_core.fw_jobctl_file_sheet_runschedule set latest_runschedule_flag='0' where latest_runschedule_flag='1';"
#inserting into fw_core.fw_jobctl_runschedule
chkrun=$(psql $psql_base_arg -t -c "select count(*) from fw_core.fw_jobctl_jobstep_master where client_id=21 and function_id=1 and run_frequency='daily' and active_flag=true;")
chkrun=$(psql $psql_base_arg -t -c "select count(*) from fw_core.fw_jobctl_jobstep_master where client_id=20 and function_id=1 and run_frequency='daily' and active_flag=true;")
if [[ $chkrun -gt 0 ]]; then
psql $psql_base_arg -q -c "INSERT INTO fw_core.fw_jobctl_runschedule
(client_id, function_id, run_schedule_timestamp, run_frequency, start_time, begin_status, created_by, updated_by, create_timestamp, update_timestamp)
@@ -52,19 +64,19 @@ where active_flag='TRUE' group by client_id, function_id, run_frequency limit 1;
#check previous day job status in fw_jobctl_runschedule
chkrun2=$(psql $psql_base_arg -t -c "select trim(end_status) from fw_core.fw_jobctl_runschedule where run_schedule_id = (
select max(run_schedule_id) as max from fw_core.fw_jobctl_runschedule where latest_runschedule_flag='0' and client_id=21 and function_id=1);")
select max(run_schedule_id) as max from fw_core.fw_jobctl_runschedule where latest_runschedule_flag='0' and client_id=20 and function_id=1);")
if [ "$chkrun2" != " error" ]; then
#Update fw_jobctl_runschedule
psql $psql_base_arg -q -c "update fw_core.fw_jobctl_runschedule set begin_status='started' where client_id=21 and function_id=1 and run_frequency='daily' and latest_runschedule_flag='1' "
psql $psql_base_arg -q -c "update fw_core.fw_jobctl_runschedule set begin_status='started' where client_id=20 and function_id=1 and run_frequency='daily' and latest_runschedule_flag='1' "
else
psql $psql_base_arg -q -c "update fw_core.fw_jobctl_runschedule set begin_status='started',end_status='error',end_status_note = 'previous day job failed' where client_id=21 and function_id=1 and run_frequency='daily' and latest_runschedule_flag='1' "
psql $psql_base_arg -q -c "update fw_core.fw_jobctl_runschedule set begin_status='started',end_status='error',end_status_note = 'previous day job failed' where client_id=20 and function_id=1 and run_frequency='daily' and latest_runschedule_flag='1' "
echo "welcome2"
exit 1;
fi
else
echo " No active job for daily for client_id=21"
echo " No active job for daily for client_id=20"
exit 1;
fi

22
scripts/onetime_compile.sh
View File

@@ -1,7 +1,21 @@
#!/bin/bash -e
HOME="/home/compegence/customer"
#/****************************************************************
#****** ForeWarn Licensing and / or usage Terms and Conditions *****
#ForeWarn. The NextGen Insights Solution Platform
#Copyright © 2021 COMPEGENCE. All Rights Reserved
#ForeWarn is a product of COMPEGENCE.
#To be used only with a valid license from COMPEGENCE
#www.compegence.com info@compegence.com
#****************************************************************/
#HOME="/home/compegence/customer"
#Directories
homeDir=$HOME/MMT
homeDir=$HOME/customer/MMT
configDir=$homeDir/conf
input_file=$configDir/etl-config.txt
declare -A credentials
@@ -19,7 +33,7 @@ server=${credentials[server]}
mmt_con="psql -X -q -A -t postgresql://$user:$password@$host/$db?options=--search_path%3D"
declare -a archivepathvar=("$homeDir/sql/onetime/archive/schema")
#declare -a archivepathvar=("$homeDir/sql/onetime/archive/schema")
declare -a archivepathsvar=("$homeDir/sql/onetime/archive/create" "$homeDir/sql/onetime/archive/function")
@@ -39,7 +53,7 @@ func_execute_conn(){
done
}
func_execute_conn "$mmt_con'archive'" archivepathvar
#func_execute_conn "$mmt_con'archive'" archivepathvar
func_execute_conn "$mmt_con'staging2'" stgpathvar

39
scripts/post_update.sh Executable file
View File

@@ -0,0 +1,39 @@
#!/bin/bash -e
#/****************************************************************
#****** ForeWarn Licensing and / or usage Terms and Conditions *****
#ForeWarn. The NextGen Insights Solution Platform
#Copyright ¿ 2021 COMPEGENCE. All Rights Reserved
#ForeWarn is a product of COMPEGENCE.
#To be used only with a valid license from COMPEGENCE
#www.compegence.com info@compegence.com
#****************************************************************/
HOME="/home/compegence/customer"
#Directories
homeDir=$HOME/MMT
configDir=$homeDir/conf
input_file=$configDir/etl-config.txt
declare -A credentials
while IFS="=" read -r key value; do
credentials[$key]=$value
done < <( sed -e '/^\s*$/ d' -e '/^#/ d' $input_file )
user=${credentials[user]}
password=${credentials[password]}
host=${credentials[host]}
db=${credentials[db]}
protocol=${credentials[protocol]}
server=${credentials[server]}
psql_base_arg="postgresql://$user:$password@$host/$db"
#update fw_jobctl_file_landing
psql $psql_base_arg -t -q -c "update fw_core.fw_jobctl_file_landing a set run_schedule_id=b.run_schedule_id,file_syspk=b.file_syspk,file_landing_date=b.file_landing_date,status=b.end_status from fw_core.fw_jobctl_file_runschedule b where a.file_name=b.file_name and b.latest_runschedule_flag=true;"

34
scripts/run_schedule.sh
View File

@@ -1,7 +1,21 @@
#!/bin/bash -e
HOME="/home/compegence/customer"
#/****************************************************************
#****** ForeWarn Licensing and / or usage Terms and Conditions *****
#ForeWarn. The NextGen Insights Solution Platform
#Copyright © 2021 COMPEGENCE. All Rights Reserved
#ForeWarn is a product of COMPEGENCE.
#To be used only with a valid license from COMPEGENCE
#www.compegence.com info@compegence.com
#****************************************************************/
#HOME="/home/compegence/customer"
#Directories
homeDir=$HOME/MMT
homeDir=$HOME/customer/MMT
configDir=$homeDir/conf
input_file=$configDir/etl-config.txt
declare -A credentials
@@ -22,7 +36,7 @@ server=${credentials[server]}
psql_base_arg="postgresql://$user:$password@$host/$db"
# load the record into the fw_jobctl_runschedule accounting for run frequency (hourly, daily, monthly, yearly) and insert
/home/compegence/customer/MMT/sql/scripts/job_load.sh
$homeDir/sql/scripts/job_load.sh
##***** Run frequency loop; Outer Loop
#while loop for each client_id,function_id,run_frequency from fw_jobctl_runschedule_jobstep
@@ -78,10 +92,14 @@ echo "Executing Job: $job_name by checking Previous End Status"
## Process java Program
if [[ $job_script_type == "java" ]]; then
psql $psql_base_arg -t -q -c "update fw_core.fw_jobctl_runschedule_jobstep set start_time=now(),begin_status='started' where job_step_script_name='$job_step_script_name' and client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' and latest_runschedule_flag='1' "
echo "--$prev_job_seq"
if [[ $prev_job_seq == '' ]]; then
java -jar $job_step_script_name
echo "calling java jar"
java -jar $job_step_script_name
else
chkrun=$(psql $psql_base_arg -t -c "select TRIM(end_status) from fw_core.fw_jobctl_runschedule_jobstep where latest_runschedule_flag='1' and job_step_run_dependency_seuqence=$prev_job_seq and client_id=21 and function_id=1;")
chkrun=$(psql $psql_base_arg -t -c "select TRIM(end_status) from fw_core.fw_jobctl_runschedule_jobstep where latest_runschedule_flag='1' and job_step_run_dependency_seuqence=$prev_job_seq and client_id=20 and function_id=1;")
echo "chk $chkrun"
if [[ "$chkrun" == " success" ]]; then
java -jar $job_step_script_name
else
@@ -95,7 +113,7 @@ echo "Executing Job: $job_name by checking Previous End Status"
if [[ $prev_job_seq == '' ]]; then
psql $psql_base_arg -a -f $job_step_script_name $host $port $user $pwd $dbname $param $client_id $function_id $data_from_date $data_to_date $job_scope $job_scope_qualifier
else
chkrun=$(psql $psql_base_arg -t -c "select TRIM(end_status) from fw_core.fw_jobctl_runschedule_jobstep where latest_runschedule_flag='1' and job_step_run_dependency_seuqence=$prev_job_seq and client_id=21 and function_id=1;")
chkrun=$(psql $psql_base_arg -t -c "select TRIM(end_status) from fw_core.fw_jobctl_runschedule_jobstep where latest_runschedule_flag='1' and job_step_run_dependency_seuqence=$prev_job_seq and client_id=20 and function_id=1;")
if [[ "$chkrun" == " success" && $file_count -gt 0 ]]; then
psql $psql_base_arg -a -f $job_step_script_name $host $port $user $pwd $dbname $param $client_id $function_id $data_from_date $data_to_date $job_scope $job_scope_qualifier
else
@@ -109,7 +127,7 @@ echo "Executing Job: $job_name by checking Previous End Status"
if [[ $prev_job_seq == '' ]]; then
$job_step_script_name
else
chkrun=$(psql $psql_base_arg -t -c "select TRIM(end_status) from fw_core.fw_jobctl_runschedule_jobstep where latest_runschedule_flag='1' and job_step_run_dependency_seuqence=$prev_job_seq and client_id=21 and function_id=1;")
chkrun=$(psql $psql_base_arg -t -c "select TRIM(end_status) from fw_core.fw_jobctl_runschedule_jobstep where latest_runschedule_flag='1' and job_step_run_dependency_seuqence=$prev_job_seq and client_id=20 and function_id=1;")
if [[ "$chkrun" == " success" && $file_count -gt 0 ]]; then
$job_step_script_name
@@ -125,7 +143,7 @@ echo "Executing Job: $job_name by checking Previous End Status"
psql $psql_base_arg -c "select $job_step_script_name_concat"
else
chkrun=$(psql $psql_base_arg -t -c "select TRIM(end_status) from fw_core.fw_jobctl_runschedule_jobstep where latest_runschedule_flag='1' and job_step_run_dependency_seuqence=$prev_job_seq and client_id=21 and function_id=1;")
chkrun=$(psql $psql_base_arg -t -c "select TRIM(end_status) from fw_core.fw_jobctl_runschedule_jobstep where latest_runschedule_flag='1' and job_step_run_dependency_seuqence=$prev_job_seq and client_id=20 and function_id=1;")
if [[ "$chkrun" == " success" && $file_count -gt 0 ]]; then
psql $psql_base_arg -t -c "select $job_step_script_name_concat"
else