latest_mmt_version
This commit is contained in:
Deepthi
3
MMT_latest_version/java/generic_table/.idea/.gitignore
generated
vendored
Normal file
3
MMT_latest_version/java/generic_table/.idea/.gitignore
generated
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
# Default ignored files
|
||||
/shelf/
|
||||
/workspace.xml
|
||||
1
MMT_latest_version/java/generic_table/.idea/.name
generated
Normal file
1
MMT_latest_version/java/generic_table/.idea/.name
generated
Normal file
@@ -0,0 +1 @@
|
||||
ex1
|
||||
15
MMT_latest_version/java/generic_table/.idea/compiler.xml
generated
Normal file
15
MMT_latest_version/java/generic_table/.idea/compiler.xml
generated
Normal file
@@ -0,0 +1,15 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="CompilerConfiguration">
|
||||
<annotationProcessing>
|
||||
<profile name="Maven default annotation processors profile" enabled="true">
|
||||
<sourceOutputDir name="target/generated-sources/annotations" />
|
||||
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
|
||||
<outputRelativeToContentRoot value="true" />
|
||||
<module name="generic_table" />
|
||||
<module name="ex1" />
|
||||
<module name="maven project" />
|
||||
</profile>
|
||||
</annotationProcessing>
|
||||
</component>
|
||||
</project>
|
||||
6
MMT_latest_version/java/generic_table/.idea/encodings.xml
generated
Normal file
6
MMT_latest_version/java/generic_table/.idea/encodings.xml
generated
Normal file
@@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="Encoding">
|
||||
<file url="file://$PROJECT_DIR$/src/main/java" charset="UTF-8" />
|
||||
</component>
|
||||
</project>
|
||||
25
MMT_latest_version/java/generic_table/.idea/jarRepositories.xml
generated
Normal file
25
MMT_latest_version/java/generic_table/.idea/jarRepositories.xml
generated
Normal file
@@ -0,0 +1,25 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="RemoteRepositoriesConfiguration">
|
||||
<remote-repository>
|
||||
<option name="id" value="central" />
|
||||
<option name="name" value="Central Repository" />
|
||||
<option name="url" value="https://repo.maven.apache.org/maven2" />
|
||||
</remote-repository>
|
||||
<remote-repository>
|
||||
<option name="id" value="AsposeJavaAPI" />
|
||||
<option name="name" value="Aspose Java API" />
|
||||
<option name="url" value="http://repository.aspose.com/repo/" />
|
||||
</remote-repository>
|
||||
<remote-repository>
|
||||
<option name="id" value="central" />
|
||||
<option name="name" value="Maven Central repository" />
|
||||
<option name="url" value="https://repo1.maven.org/maven2" />
|
||||
</remote-repository>
|
||||
<remote-repository>
|
||||
<option name="id" value="jboss.community" />
|
||||
<option name="name" value="JBoss Community repository" />
|
||||
<option name="url" value="https://repository.jboss.org/nexus/content/repositories/public/" />
|
||||
</remote-repository>
|
||||
</component>
|
||||
</project>
|
||||
14
MMT_latest_version/java/generic_table/.idea/misc.xml
generated
Normal file
14
MMT_latest_version/java/generic_table/.idea/misc.xml
generated
Normal file
@@ -0,0 +1,14 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ExternalStorageConfigurationManager" enabled="true" />
|
||||
<component name="MavenProjectsManager">
|
||||
<option name="originalFiles">
|
||||
<list>
|
||||
<option value="$PROJECT_DIR$/pom.xml" />
|
||||
</list>
|
||||
</option>
|
||||
</component>
|
||||
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" project-jdk-name="1.8" project-jdk-type="JavaSDK">
|
||||
<output url="file://$PROJECT_DIR$/out" />
|
||||
</component>
|
||||
</project>
|
||||
@@ -0,0 +1,67 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.example</groupId>
|
||||
<artifactId>ex1</artifactId>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
<build>
|
||||
<pluginManagement>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-shade-plugin</artifactId>
|
||||
<version>3.2.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>shade</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<transformers>
|
||||
<transformer>
|
||||
<mainClass>org.examle.App</mainClass>
|
||||
</transformer>
|
||||
</transformers>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<createDependencyReducedPom>true</createDependencyReducedPom>
|
||||
<filters>
|
||||
<filter>
|
||||
<artifact>*:*</artifact>
|
||||
<excludes>
|
||||
<exclude>META-INF/*.SF</exclude>
|
||||
<exclude>META-INF/*.DSA</exclude>
|
||||
<exclude>META-INF/*.RSA</exclude>
|
||||
</excludes>
|
||||
</filter>
|
||||
</filters>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</pluginManagement>
|
||||
</build>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>4.11</version>
|
||||
<scope>test</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<artifactId>hamcrest-core</artifactId>
|
||||
<groupId>org.hamcrest</groupId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<properties>
|
||||
<postgresql.version>42.1.4</postgresql.version>
|
||||
<mainClass>org.example.App</mainClass>
|
||||
<maven.compiler.source>1.8</maven.compiler.source>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<start-class>org.example.App</start-class>
|
||||
<maven.compiler.target>1.8</maven.compiler.target>
|
||||
</properties>
|
||||
</project>
|
||||
2
MMT_latest_version/java/generic_table/ex1.iml
Normal file
2
MMT_latest_version/java/generic_table/ex1.iml
Normal file
@@ -0,0 +1,2 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module type="JAVA_MODULE" version="4" />
|
||||
Binary file not shown.
Binary file not shown.
94
MMT_latest_version/java/generic_table/pom.xml
Normal file
94
MMT_latest_version/java/generic_table/pom.xml
Normal file
@@ -0,0 +1,94 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.example</groupId>
|
||||
<artifactId>ex1</artifactId>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<maven.compiler.source>1.8</maven.compiler.source>
|
||||
<maven.compiler.target>1.8</maven.compiler.target>
|
||||
<postgresql.version>42.1.4</postgresql.version>
|
||||
<mainClass>org.example.App</mainClass>
|
||||
<start-class>org.example.App</start-class>
|
||||
</properties>
|
||||
|
||||
<packaging>jar</packaging>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi</artifactId>
|
||||
<version>4.0.1</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml -->
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi-ooxml</artifactId>
|
||||
<version>4.0.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-lang3</artifactId>
|
||||
<version>3.1</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>4.11</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.postgresql</groupId>
|
||||
<artifactId>postgresql</artifactId>
|
||||
<version>${postgresql.version}</version>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
<build>
|
||||
<pluginManagement><!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->
|
||||
<plugins>
|
||||
<!-- <plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-jar-plugin</artifactId>
|
||||
<configuration>
|
||||
<archive>
|
||||
<manifest>
|
||||
<addClasspath>true</addClasspath>
|
||||
<mainClass>org.example.App</mainClass>
|
||||
</manifest>
|
||||
</archive>
|
||||
</configuration>
|
||||
</plugin> -->
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-shade-plugin</artifactId>
|
||||
<version>3.2.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>shade</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<transformers>
|
||||
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
|
||||
<mainClass>org.example.App</mainClass>
|
||||
</transformer>
|
||||
</transformers>
|
||||
<shadedArtifactAttached>true</shadedArtifactAttached>
|
||||
<!-- <shadedClassifierName>launcher</shadedClassifierName> -->
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</pluginManagement>
|
||||
</build>
|
||||
</project>
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
295
MMT_latest_version/java/generic_table/src/main/java/example.java
Normal file
295
MMT_latest_version/java/generic_table/src/main/java/example.java
Normal file
@@ -0,0 +1,295 @@
|
||||
package org.example;
|
||||
|
||||
import java.io.*;
|
||||
import java.sql.*;
|
||||
import java.util.*;
|
||||
import java.util.Date;
|
||||
|
||||
import org.apache.poi.ss.usermodel.*;
|
||||
import org.apache.poi.xssf.usermodel.*;
|
||||
|
||||
/**
|
||||
* Sample Java program that imports data from an Excel file to MySQL database.
|
||||
*
|
||||
* @author Nam Ha Minh - https://www.codejava.net
|
||||
*
|
||||
*/
|
||||
public class example {
|
||||
|
||||
public static void main(String[] args) {
|
||||
String jdbcURL = "jdbc:mysql://localhost:3306/sales";
|
||||
String username = "user";
|
||||
String password = "password";
|
||||
|
||||
String excelFilePath = "Students.xlsx";
|
||||
|
||||
int batchSize = 20;
|
||||
|
||||
Connection connection = null;
|
||||
|
||||
try {
|
||||
long start = System.currentTimeMillis();
|
||||
|
||||
FileInputStream inputStream = new FileInputStream(excelFilePath);
|
||||
|
||||
Workbook workbook = new XSSFWorkbook(inputStream);
|
||||
|
||||
Sheet firstSheet = workbook.getSheetAt(0);
|
||||
Iterator<Row> rowIterator = firstSheet.iterator();
|
||||
|
||||
connection = DriverManager.getConnection(jdbcURL, username, password);
|
||||
connection.setAutoCommit(false);
|
||||
|
||||
String sql = "INSERT INTO students (name, enrolled, progress) VALUES (?, ?, ?)";
|
||||
PreparedStatement statement = connection.prepareStatement(sql);
|
||||
|
||||
int count = 0;
|
||||
|
||||
rowIterator.next(); // skip the header row
|
||||
|
||||
while (rowIterator.hasNext()) {
|
||||
Row nextRow = rowIterator.next();
|
||||
Iterator<Cell> cellIterator = nextRow.cellIterator();
|
||||
|
||||
while (cellIterator.hasNext()) {
|
||||
Cell nextCell = cellIterator.next();
|
||||
|
||||
int columnIndex = nextCell.getColumnIndex();
|
||||
|
||||
switch (columnIndex) {
|
||||
case 0:
|
||||
String name = nextCell.getStringCellValue();
|
||||
statement.setString(1, name);
|
||||
break;
|
||||
case 1:
|
||||
Date enrollDate = nextCell.getDateCellValue();
|
||||
statement.setTimestamp(2, new Timestamp(enrollDate.getTime()));
|
||||
case 2:
|
||||
int progress = (int) nextCell.getNumericCellValue();
|
||||
statement.setInt(3, progress);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
statement.addBatch();
|
||||
|
||||
if (count % batchSize == 0) {
|
||||
statement.executeBatch();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
workbook.close();
|
||||
|
||||
// execute the remaining queries
|
||||
statement.executeBatch();
|
||||
|
||||
connection.commit();
|
||||
connection.close();
|
||||
|
||||
long end = System.currentTimeMillis();
|
||||
System.out.printf("Import done in %d ms\n", (end - start));
|
||||
|
||||
} catch (IOException ex1) {
|
||||
System.out.println("Error reading file");
|
||||
ex1.printStackTrace();
|
||||
} catch (SQLException ex2) {
|
||||
System.out.println("Database error");
|
||||
ex2.printStackTrace();
|
||||
}
|
||||
|
||||
}
|
||||
public String loadSheetInDatabase(String inputFile) {
|
||||
try {
|
||||
XSSFWorkbook wb = new XSSFWorkbook(new FileInputStream(inputFile));
|
||||
|
||||
// Get first sheet from the workbook
|
||||
XSSFSheet sheet = wb.getSheetAt(0);
|
||||
|
||||
Row row;
|
||||
Cell cell;
|
||||
|
||||
// Iterate through each rows from first sheet
|
||||
Iterator < Row > rowIterator = sheet.iterator();
|
||||
|
||||
Boolean parsedHeaders = false;
|
||||
List < String > headers = new ArrayList < String > ();
|
||||
List < String > rowValues = null;
|
||||
while (rowIterator.hasNext()) {
|
||||
row = rowIterator.next();
|
||||
if (parsedHeaders) {
|
||||
rowValues = new ArrayList <String> ();
|
||||
}
|
||||
// System.out.println ("Row No.: " + row.getRowNum ());
|
||||
// For each row, iterate through each columns
|
||||
Iterator < Cell > cellIterator = row.cellIterator();
|
||||
|
||||
while (cellIterator.hasNext()) {
|
||||
cell = cellIterator.next();
|
||||
if (!parsedHeaders) {
|
||||
headers.add(cell.getStringCellValue());
|
||||
} else {
|
||||
rowValues.add(cell.getStringCellValue());
|
||||
}
|
||||
}
|
||||
if(!parsedHeaders){
|
||||
parsedHeaders = true;
|
||||
}else{
|
||||
String rowValuesString = "";
|
||||
for(String fieldValue : rowValues){
|
||||
rowValuesString += fieldValue + " ";
|
||||
}
|
||||
System.out.println(rowValuesString);
|
||||
InsertRowInDB(headers, rowValues);
|
||||
System.out.println("");
|
||||
}
|
||||
}
|
||||
} catch (FileNotFoundException e) {
|
||||
e.printStackTrace();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
} catch (SQLException throwables) {
|
||||
throwables.printStackTrace();
|
||||
} catch (ClassNotFoundException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return "success";
|
||||
}
|
||||
|
||||
public void InsertRowInDB(List<String> headers, List<String> values) throws SQLException, ClassNotFoundException {
|
||||
|
||||
Class.forName("org.postgresql.Driver");
|
||||
System.out.println("connected");
|
||||
Connection conn = DriverManager.getConnection("jdbc:postgresql:postgres", "postgres", "sridevi");
|
||||
|
||||
String insertColumns = "";
|
||||
String preparedStatementClause = "";
|
||||
for(int i = 0; i < headers.size(); i++){
|
||||
String fieldValue = headers.get(i);
|
||||
insertColumns += fieldValue;
|
||||
preparedStatementClause += "?";
|
||||
if(i <= (headers.size() - 1)){
|
||||
insertColumns += ", ";
|
||||
preparedStatementClause += ", ";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
String query = String.format( "Insert into excel(%s) values(%s)" , insertColumns, preparedStatementClause);
|
||||
PreparedStatement ps = conn.prepareStatement(query);
|
||||
for(int i = 0; i < values.size(); i++){
|
||||
ps.setString(i, values.get(i));
|
||||
}
|
||||
ps.executeUpdate();
|
||||
System.out.println("Values Inserted Successfully");
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
package org.example;
|
||||
import java.io.*;
|
||||
import java.sql.*;
|
||||
import java.util.*;
|
||||
import java.util.Date;
|
||||
|
||||
import org.apache.poi.ss.usermodel.*;
|
||||
import org.apache.poi.xssf.usermodel.*;
|
||||
/**
|
||||
* Hello world!
|
||||
*
|
||||
|
||||
public class App {
|
||||
public static void main(String[] args) {
|
||||
System.out.println("Hello World!");
|
||||
|
||||
String excelFilePath = "D:\\maven project\\src\\main\\java\\Arjun Novo - Rotavator.xlsx";
|
||||
loadSheetInDatabase(excelFilePath);
|
||||
}
|
||||
|
||||
public static String loadSheetInDatabase(String inputFile) {
|
||||
try {
|
||||
XSSFWorkbook wb = new XSSFWorkbook(new FileInputStream(inputFile));
|
||||
|
||||
// Get first sheet from the workbook
|
||||
XSSFSheet sheet = wb.getSheetAt(0);
|
||||
|
||||
Row row;
|
||||
Cell cell;
|
||||
|
||||
// Iterate through each rows from first sheet
|
||||
Iterator<Row> rowIterator = sheet.iterator();
|
||||
|
||||
Boolean parsedHeaders = false;
|
||||
List<String> headers = new ArrayList<String>();
|
||||
List<String> rowValues = null;
|
||||
while (rowIterator.hasNext()) {
|
||||
row = rowIterator.next();
|
||||
if (parsedHeaders) {
|
||||
rowValues = new ArrayList<String>();
|
||||
}
|
||||
// System.out.println ("Row No.: " + row.getRowNum ());
|
||||
// For each row, iterate through each columns
|
||||
Iterator<Cell> cellIterator = row.cellIterator();
|
||||
|
||||
while (cellIterator.hasNext()) {
|
||||
cell = cellIterator.next();
|
||||
if (!parsedHeaders) {
|
||||
headers.add(cell.getStringCellValue());
|
||||
} else {
|
||||
rowValues.add(cell.getStringCellValue());
|
||||
}
|
||||
}
|
||||
if (!parsedHeaders) {
|
||||
parsedHeaders = true;
|
||||
} else {
|
||||
String rowValuesString = "";
|
||||
for (String fieldValue : rowValues) {
|
||||
rowValuesString += fieldValue + " ";
|
||||
}
|
||||
System.out.println(rowValuesString);
|
||||
InsertRowInDB(headers, rowValues);
|
||||
System.out.println("");
|
||||
}
|
||||
}
|
||||
} catch (FileNotFoundException e) {
|
||||
e.printStackTrace();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
} catch (SQLException throwables) {
|
||||
throwables.printStackTrace();
|
||||
} catch (ClassNotFoundException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return "success";
|
||||
}
|
||||
|
||||
public static void InsertRowInDB(List<String> headers, List<String> values) throws SQLException, ClassNotFoundException {
|
||||
|
||||
Class.forName("org.postgresql.Driver");
|
||||
System.out.println("connected");
|
||||
Connection conn = DriverManager.getConnection("jdbc:postgresql:postgres", "postgres", "postgres");
|
||||
|
||||
String insertColumns = "";
|
||||
String preparedStatementClause = "";
|
||||
for(int i = 0; i < headers.size(); i++){
|
||||
String fieldValue = headers.get(i);
|
||||
insertColumns += fieldValue;
|
||||
preparedStatementClause += "?";
|
||||
if(i <= (headers.size() - 1)){
|
||||
insertColumns += ", ";
|
||||
preparedStatementClause += ", ";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
String query = String.format( "Insert into excel(%s) values(%s)" , insertColumns, preparedStatementClause);
|
||||
PreparedStatement ps = conn.prepareStatement(query);
|
||||
for(int i = 0; i < values.size(); i++){
|
||||
ps.setString(i, values.get(i));
|
||||
}
|
||||
ps.executeUpdate();
|
||||
System.out.println("Values Inserted Successfully");
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,615 @@
|
||||
package org.example;
|
||||
import java.io.*;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.sql.*;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.*;
|
||||
|
||||
import org.apache.poi.openxml4j.util.ZipSecureFile;
|
||||
import org.apache.poi.ss.usermodel.*;
|
||||
import org.apache.poi.ss.usermodel.Row.MissingCellPolicy;
|
||||
import org.apache.poi.ss.util.CellRangeAddress;
|
||||
import org.apache.poi.xssf.usermodel.*;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.FormulaEvaluator;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.xmlbeans.impl.values.XmlValueDisconnectedException;
|
||||
import java.nio.file.*;
|
||||
import java.nio.file.attribute.*;
|
||||
import java.util.logging.Logger;
|
||||
import java.util.logging.Level;
|
||||
|
||||
public class App {
|
||||
private String sourceFile;
|
||||
private String outputFile;
|
||||
private XSSFWorkbook workbook;
|
||||
private static XSSFCell Cell;
|
||||
private static XSSFRow Row;
|
||||
private static MissingCellPolicy xRow;
|
||||
|
||||
private final static Logger LOGGER =
|
||||
Logger.getLogger(Logger.GLOBAL_LOGGER_NAME);
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
||||
// private static Logger logger = Logger.getLogger(MyClass.class);
|
||||
FileFilter filter = new ExcelFileFilter();
|
||||
|
||||
// File src_directory = new File("D:\\customer\\MMT\\files\\landing");
|
||||
File src_directory = new File("/home/compegence/customer/MMT/files/landing");
|
||||
|
||||
//File destination = new File("D:\\customer\\MMT\\files\\processed");
|
||||
File destination = new File("/home/compegence/customer/MMT/files/processed");
|
||||
|
||||
//Exception:org.apache.poi.ooxml.POIXMLException: Zip bomb detected!
|
||||
ZipSecureFile.setMinInflateRatio(0);
|
||||
|
||||
String db = "",host = "",username="",password="",port="";
|
||||
String[] split_arr=new String[100];
|
||||
String [] codes = new String[100];
|
||||
int c=0;
|
||||
// String HomeDir = System.getProperty("user.home");
|
||||
Scanner input = null;
|
||||
|
||||
App app = new App();
|
||||
|
||||
try {
|
||||
input = new Scanner(new File("/home/compegence/customer/MMT/conf/etl-config.txt"));
|
||||
//input = new Scanner(new File("etlconfig.txt"));
|
||||
while(input.hasNext()) {
|
||||
codes[c] = input.nextLine();
|
||||
|
||||
if(codes[c].contains("db"))
|
||||
{
|
||||
split_arr = codes[c].split("=");
|
||||
db=split_arr[1];
|
||||
}
|
||||
if(codes[c].contains("host"))
|
||||
{
|
||||
split_arr = codes[c].split("=");
|
||||
host=split_arr[1];
|
||||
}
|
||||
if(codes[c].contains("user"))
|
||||
{
|
||||
split_arr = codes[c].split("=");
|
||||
username=split_arr[1];
|
||||
}
|
||||
if(codes[c].contains("password"))
|
||||
{
|
||||
split_arr = codes[c].split("=");
|
||||
password=split_arr[1];
|
||||
}
|
||||
if(codes[c].contains("port"))
|
||||
{
|
||||
split_arr = codes[c].split("=");
|
||||
port=split_arr[1];
|
||||
}
|
||||
c++;
|
||||
}
|
||||
|
||||
} catch (FileNotFoundException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
//db="m_test";
|
||||
//String jdbcURL = "jdbc:postgresql://localhost:2345/m_test?currentSchema=staging1";
|
||||
String jdbcURL = "jdbc:postgresql://"+host+":"+port+"/"+db+"?currentSchema=staging1";
|
||||
|
||||
// String username = "postgres";
|
||||
// String password = "j3(jLBq}e"; // String username = null,password = null, jdbcURL = null;
|
||||
|
||||
Connection connection = null;
|
||||
Statement stmt = null;
|
||||
|
||||
String jdbcURL2 = "jdbc:postgresql://localhost:2345/fw_demo?currentSchema=fw_core";
|
||||
Connection connection2 = null;
|
||||
|
||||
int client_id=0,function_id=0,job_id=0,step_id=0, max_run_schedule_id=0;
|
||||
DataFormatter formatter = new DataFormatter(true);
|
||||
|
||||
if (src_directory.isDirectory()) {
|
||||
File[] files = src_directory.listFiles(filter);
|
||||
|
||||
if (src_directory.length() > 0) {
|
||||
// if (src_directory.length() >= 0) {
|
||||
|
||||
try {
|
||||
connection2 = DriverManager.getConnection(jdbcURL2, username, password);
|
||||
connection2.setAutoCommit(false);
|
||||
|
||||
//set connection for mmt
|
||||
connection = DriverManager.getConnection(jdbcURL, username, password);
|
||||
connection.setAutoCommit(false);
|
||||
|
||||
java.sql.Timestamp run_schedule_date = java.sql.Timestamp.valueOf("1900-01-01 00:00:00");
|
||||
|
||||
//Pick up the client_id, function_id and run_schedule_id
|
||||
stmt = connection.createStatement();
|
||||
String qry0 = "select client_id,function_id,run_schedule_id,run_schedule_timestamp,job_id,step_id from fw_core.fw_jobctl_runschedule_jobstep where latest_runschedule_flag='1' and job_script_type='java' ;";
|
||||
ResultSet rs0 = stmt.executeQuery(qry0);
|
||||
while (rs0.next()) {
|
||||
client_id = rs0.getInt("client_id");
|
||||
function_id = rs0.getInt("function_id");
|
||||
max_run_schedule_id = rs0.getInt("run_schedule_id");
|
||||
run_schedule_date = rs0.getTimestamp("run_schedule_timestamp");
|
||||
job_id = rs0.getInt("job_id");
|
||||
step_id = rs0.getInt("step_id");
|
||||
}
|
||||
rs0.close();
|
||||
|
||||
// java.sql.Timestamp file_created_date1 = new java.sql.Timestamp(run_schedule_date.getTime());
|
||||
|
||||
int max_file_syspk = 1;
|
||||
int file_count = 0;
|
||||
|
||||
for (File file : files) {
|
||||
try {
|
||||
System.out.println("run_schedule_date2"+run_schedule_date);
|
||||
|
||||
file_count++;
|
||||
Path path = Paths.get(file.getPath());
|
||||
BasicFileAttributes attr;
|
||||
//init workbook and do stuff
|
||||
|
||||
//Extracting 2nd word from file name
|
||||
String[] file_name_split_array = file.getName().split("_", 3);
|
||||
String file_format = file_name_split_array[1];
|
||||
// String file_name = file_name_split_array[2];
|
||||
String file_name = file.getName();
|
||||
|
||||
//String to date conversion
|
||||
String startDate = file_name_split_array[0];//"01-02-2013";
|
||||
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
|
||||
java.util.Date date = sdf.parse(startDate);
|
||||
java.sql.Date fileDate = new java.sql.Date(date.getTime());
|
||||
|
||||
//Reading file creation time from directory
|
||||
attr = Files.readAttributes(path, BasicFileAttributes.class);
|
||||
FileTime fileTime = attr.creationTime();
|
||||
java.util.Date utilDate = new java.util.Date(fileTime.toMillis());
|
||||
java.sql.Timestamp file_created_date = new java.sql.Timestamp(utilDate.getTime());
|
||||
|
||||
//Get max file_syspk from mmt_ods.fw_jobctl_file_runschedule
|
||||
stmt = connection.createStatement();
|
||||
String qry1 = "select max(file_syspk) as max_file_syspk from fw_core.fw_jobctl_file_runschedule;";
|
||||
ResultSet rs1 = stmt.executeQuery(qry1);
|
||||
while (rs1.next()) {
|
||||
max_file_syspk = rs1.getInt("max_file_syspk");
|
||||
}
|
||||
rs1.close();
|
||||
max_file_syspk++;
|
||||
|
||||
//insert record for each file in fw_jobctl_file_runschedule
|
||||
String sql_file_level = " INSERT INTO fw_core.fw_jobctl_file_runschedule(client_id, function_id,latest_runschedule_flag,run_schedule_id,job_id,step_id, file_syspk,file_name, file_mnemonic, file_landing_date,staging_type, begin_status, start_time, created_by, updated_by, create_timestamp, update_timestamp)" +
|
||||
"VALUES(?,?,'1',?,?,?,?,?,?,?,?,'Started',now(), 'Admin','Admin',now(), now());";
|
||||
PreparedStatement stmt1 = connection.prepareStatement(sql_file_level);
|
||||
|
||||
stmt1.setInt(1, client_id);
|
||||
stmt1.setInt(2, function_id);
|
||||
stmt1.setInt(3, max_run_schedule_id);
|
||||
stmt1.setInt(4, job_id);
|
||||
stmt1.setInt(5, step_id);
|
||||
stmt1.setInt(6, max_file_syspk);
|
||||
stmt1.setString(7, file_name);
|
||||
stmt1.setString(8, file_format);
|
||||
stmt1.setTimestamp(9,file_created_date);
|
||||
stmt1.setString(10,"stg1");
|
||||
|
||||
stmt1.executeUpdate();
|
||||
|
||||
System.out.println("max_file_syspk: " + max_file_syspk);
|
||||
long start = System.currentTimeMillis();
|
||||
|
||||
FileInputStream inputStream = new FileInputStream(file);
|
||||
|
||||
Workbook workbook = new XSSFWorkbook(inputStream);
|
||||
// Workbook workbook = WorkbookFactory.create(file);
|
||||
|
||||
for (int k = 0; k < workbook.getNumberOfSheets(); k++) {
|
||||
try{
|
||||
Sheet sheet1 = workbook.getSheetAt(k);
|
||||
String sheetName = workbook.getSheetAt(k).getSheetName();
|
||||
|
||||
// if (file_format.equals("FTDRY") & ( ! sheetName.equals("Tractor specifications") | ! sheetName.equals("Summary"))){
|
||||
/* if (file_format.equals("FTDRY") & ( (! sheetName.equals("Tractor specifications")) | (! sheetName.equals("Summary")))){
|
||||
|
||||
System.out.println("sheet name summary:" + sheet1.getSheetName());
|
||||
// break;
|
||||
k++;
|
||||
}*/
|
||||
//insert record for each sheet of a file in fw_jobctl_file_sheet_runschedule
|
||||
String sql_sheet_level = " INSERT INTO fw_core.fw_jobctl_file_sheet_runschedule(client_id, function_id,latest_runschedule_flag, run_schedule_id,job_id, step_id," +
|
||||
" file_syspk, sheet_id,sheet_name, file_mnemonic,staging_type, begin_status, start_time, created_by, updated_by, create_timestamp, update_timestamp)" +
|
||||
"VALUES(?,?,'1',?,?,?,?,?,?,?,?,'Started',now(), 'Admin','Admin',now(), now());";
|
||||
PreparedStatement stmt3 = connection.prepareStatement(sql_sheet_level);
|
||||
|
||||
stmt3.setInt(1, client_id);
|
||||
stmt3.setInt(2, function_id);
|
||||
stmt3.setInt(3, max_run_schedule_id);
|
||||
stmt3.setInt(4, job_id);
|
||||
stmt3.setInt(5, step_id);
|
||||
stmt3.setInt(6, max_file_syspk);
|
||||
stmt3.setInt(7, k);
|
||||
stmt3.setString(8, sheetName);
|
||||
stmt3.setString(9, file_format);
|
||||
stmt3.setString(10, "stg1");
|
||||
stmt3.executeUpdate();
|
||||
|
||||
String[] column = new String[55];//new String[sheet1.getRow(1).getPhysicalNumberOfCells() + 1];
|
||||
System.out.println("sheet name:" + sheet1.getSheetName());
|
||||
|
||||
for (int i = 0; i <= sheet1.getLastRowNum(); i++) {
|
||||
Row row = sheet1.getRow(i);
|
||||
if (isRowEmpty(row)) {
|
||||
continue;
|
||||
}
|
||||
for (int j = 0; j < sheet1.getRow(i).getPhysicalNumberOfCells() + 1; j++) {
|
||||
Cell cell = row.getCell(j);
|
||||
if ((cell == null) || (cell.equals(""))) {
|
||||
continue;
|
||||
}
|
||||
|
||||
//Removing array formula and setting modified formula. This error exists only in summary sheet
|
||||
if (cell.isPartOfArrayFormulaGroup()) {
|
||||
CellRangeAddress arrayRange = cell.getArrayFormulaRange();
|
||||
if (arrayRange.getNumberOfCells() > 1) {
|
||||
String formula = row.getCell(j).getCellFormula();
|
||||
|
||||
sheet1.getRow(i).getSheet().removeArrayFormula(cell);
|
||||
String[] formula1 = formula.split(":", 2);
|
||||
|
||||
row.getCell(j).setCellFormula(formula1[0]);
|
||||
System.out.println("Hello:" + formula1[0]);
|
||||
}
|
||||
}
|
||||
|
||||
FormulaEvaluator formulaEval = workbook.getCreationHelper().createFormulaEvaluator();
|
||||
try {
|
||||
switch (row.getCell(j).getCellType()) {
|
||||
case STRING:
|
||||
// cell.setCellType(CellType.STRING);
|
||||
row.getCell(j).setCellType(CellType.STRING);
|
||||
System.out.println("string: " + cell.getRichStringCellValue().getString());
|
||||
break;
|
||||
case NUMERIC:
|
||||
//cell.setCellType(CellType.STRING);
|
||||
row.getCell(j).setCellType(CellType.STRING);
|
||||
//System.out.println("Numeric" + cell.getRichStringCellValue().getString());
|
||||
break;
|
||||
case BOOLEAN:
|
||||
// cell.setCellType(CellType.STRING);
|
||||
row.getCell(j).setCellType(CellType.STRING);
|
||||
break;
|
||||
case FORMULA:
|
||||
row.getCell(j).setCellType(CellType.STRING);
|
||||
System.out.println("formula: " + cell.getRichStringCellValue().getString());
|
||||
break;
|
||||
case BLANK:
|
||||
// row.getCell(j).setCellType(CellType.STRING);
|
||||
break;
|
||||
case _NONE:
|
||||
// row.getCell(j).setCellType(CellType.STRING);
|
||||
break;
|
||||
case ERROR:
|
||||
// row.getCell(j).setCellType(CellType.STRING);
|
||||
break;
|
||||
default:
|
||||
System.out.println("Default");
|
||||
}
|
||||
} catch (XmlValueDisconnectedException e) {
|
||||
String v = formatter.formatCellValue(sheet1.getRow(i).getCell(j), formulaEval);
|
||||
row.getCell(j).setCellValue(v);
|
||||
System.out.println("tested after:" + v);
|
||||
}
|
||||
|
||||
column[j] = row.getCell(j).getStringCellValue();
|
||||
}
|
||||
|
||||
//inserting each line of a sheet in staging_generic_table
|
||||
String sql = "INSERT INTO staging1.staging_generic_table (file_syspk,file_name,file_prefix_date,file_mnemonic,sheet_id,sheet_name,row_number,file_creation_date,column1, column2, column3,column4,column5,column6,column7,column8,column9,column10,column11,column12,column13,column14,column15,column16,column17,column18,column19,column20,column21,column22,column23,column24," +
|
||||
"column25,column26,column27,column28,column29,column30,column31,column32,column33,column34,column35,column36,column37,column38,column39,column40,column41,column42,column43,column44,column45,column46,column47,column48,column49,column50,column51,column52,column53)" +
|
||||
" VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
|
||||
|
||||
PreparedStatement statement = connection.prepareStatement(sql);
|
||||
|
||||
statement.setInt(1, max_file_syspk);
|
||||
statement.setString(2, file.getName());
|
||||
statement.setDate(3, fileDate);
|
||||
statement.setString(4, file_format);
|
||||
statement.setInt(5, k);
|
||||
statement.setString(6, sheetName);
|
||||
statement.setInt(7, i);
|
||||
statement.setTimestamp(8, file_created_date);
|
||||
statement.setString(9, column[0]);
|
||||
statement.setString(10, column[1]);
|
||||
statement.setString(11, column[2]);
|
||||
statement.setString(12, column[3]);
|
||||
statement.setString(13, column[4]);
|
||||
statement.setString(14, column[5]);
|
||||
statement.setString(15, column[6]);
|
||||
statement.setString(16, column[7]);
|
||||
statement.setString(17, column[8]);
|
||||
statement.setString(18, column[9]);
|
||||
statement.setString(19, column[10]);
|
||||
statement.setString(20, column[11]);
|
||||
statement.setString(21, column[12]);
|
||||
statement.setString(22, column[13]);
|
||||
statement.setString(23, column[14]);
|
||||
statement.setString(24, column[15]);
|
||||
statement.setString(25, column[16]);
|
||||
statement.setString(26, column[17]);
|
||||
statement.setString(27, column[18]);
|
||||
statement.setString(28, column[19]);
|
||||
statement.setString(29, column[20]);
|
||||
statement.setString(30, column[21]);
|
||||
statement.setString(31, column[22]);
|
||||
statement.setString(32, column[23]);
|
||||
statement.setString(33, column[24]);
|
||||
statement.setString(34, column[25]);
|
||||
statement.setString(35, column[26]);
|
||||
statement.setString(36, column[27]);
|
||||
statement.setString(37, column[28]);
|
||||
statement.setString(38, column[29]);
|
||||
statement.setString(39, column[30]);
|
||||
statement.setString(40, column[31]);
|
||||
statement.setString(41, column[32]);
|
||||
statement.setString(42, column[33]);
|
||||
statement.setString(43, column[34]);
|
||||
statement.setString(44, column[35]);
|
||||
statement.setString(45, column[36]);
|
||||
statement.setString(46, column[37]);
|
||||
statement.setString(47, column[38]);
|
||||
statement.setString(48, column[39]);
|
||||
statement.setString(49, column[40]);
|
||||
statement.setString(50, column[41]);
|
||||
statement.setString(51, column[42]);
|
||||
statement.setString(52, column[43]);
|
||||
statement.setString(53, column[44]);
|
||||
statement.setString(54, column[45]);
|
||||
statement.setString(55, column[46]);
|
||||
statement.setString(56, column[47]);
|
||||
statement.setString(57, column[48]);
|
||||
statement.setString(58, column[49]);
|
||||
statement.setString(59, column[50]);
|
||||
statement.setString(60, column[51]);
|
||||
statement.setString(61, column[52]);
|
||||
|
||||
statement.executeUpdate();
|
||||
}
|
||||
|
||||
//updating if sheet loaded successfully in fw_jobctl_file_sheet_runschedule
|
||||
String sql_sheet_level2 = " update fw_core.fw_jobctl_file_sheet_runschedule set end_status='success',end_status_note='stg1_completed' , end_time=now() where file_syspk=? and sheet_id=? ";
|
||||
PreparedStatement stmt4 = connection.prepareStatement(sql_sheet_level2);
|
||||
|
||||
stmt4.setInt(1, max_file_syspk);
|
||||
stmt4.setInt(2, k);
|
||||
stmt4.executeUpdate();
|
||||
|
||||
connection.commit();
|
||||
|
||||
long end = System.currentTimeMillis();
|
||||
System.out.printf("Import done in %d ms\n", (end - start));
|
||||
|
||||
}catch (Exception e) {
|
||||
String sheet_err_msg = "file_syspk-"+ max_file_syspk + "sheet_id:"+k+ ":"+ e.toString();
|
||||
//inserting error message
|
||||
CallableStatement errorlogFunc = connection2.prepareCall("{ call fw_insert_app_error(?,?,null,?,null,null,null) }");
|
||||
errorlogFunc.setInt(1, client_id);
|
||||
errorlogFunc.setInt(2, function_id);
|
||||
errorlogFunc.setString(3, sheet_err_msg);
|
||||
errorlogFunc.execute();
|
||||
errorlogFunc.close();
|
||||
//updating sheet level failure
|
||||
String sql_sheet_level2 = " update fw_core.fw_jobctl_file_sheet_runschedule set end_status='error',end_status_note='stg1_failed' , end_time=now() where file_syspk=? and sheet_id=? ";
|
||||
PreparedStatement stmt8 = connection.prepareStatement(sql_sheet_level2);
|
||||
stmt8.setInt(1, max_file_syspk);
|
||||
stmt8.setInt(2, k);
|
||||
stmt8.executeUpdate();
|
||||
|
||||
continue;
|
||||
}
|
||||
}
|
||||
workbook.close();
|
||||
|
||||
//Move the file after reading
|
||||
inputStream.close(); //close before moving the file stream
|
||||
Path source = Paths.get(src_directory.toPath() + "//" + file.getName());
|
||||
Path target = Paths.get(destination.toPath() + "//" + file.getName());
|
||||
// java.nio.file.Files.move(Paths.get(src_directory.toPath()+"//"+file.getName()), Paths.get(destination.toPath()+"//"+file.getName()),StandardCopyOption.REPLACE_EXISTING);
|
||||
// java.nio.file.Files.move(source, target);
|
||||
|
||||
//updating if file loaded successfully in fw_jobctl_file_runschedule
|
||||
// String sql_file_level1 = " update fw_core.fw_jobctl_file_runschedule set end_status='success',end_status_note='stg1_completed', end_time=now() where file_syspk=? ";
|
||||
String sql_file_level1 = " update fw_core.fw_jobctl_file_runschedule a "+
|
||||
"set end_status = (case when t2.end_status='error' then 'error' when t2.end_status='success' then 'success' end),"+
|
||||
" end_status_note = case when t2.end_status='error' then 'stg1_failed' when t2.end_status='success' then 'stg1_completed' end "+
|
||||
"from (select distinct file_syspk,end_status from fw_core.fw_jobctl_file_sheet_runschedule where file_syspk=?) t2 "+
|
||||
"where a.file_syspk = t2.file_syspk and a.run_schedule_id=?;";
|
||||
PreparedStatement stmt2 = connection.prepareStatement(sql_file_level1);
|
||||
stmt2.setInt(1, max_file_syspk);
|
||||
stmt2.setInt(2, max_run_schedule_id);
|
||||
stmt2.executeUpdate();
|
||||
|
||||
//NA updates
|
||||
CallableStatement cleanFunc = connection.prepareCall("{ call fw_core.fn_update_na(?) }");
|
||||
cleanFunc.setInt(1, max_file_syspk);
|
||||
cleanFunc.execute();
|
||||
cleanFunc.close();
|
||||
|
||||
//update sheet_mnemonic
|
||||
String sheet_mnemonic_update = " update staging1.staging_generic_table a set sheet_format_mnemonic = b.sheet_format,file_sheet_mnemonic=b.file_sheet_mnemonic"+
|
||||
" from fw_core.test_file_sheet_format_master b where a.file_syspk =? and a.file_mnemonic=b.file_mnemonic and trim(lower(a.sheet_name)) = lower(b.sheet_name) ;";
|
||||
PreparedStatement stmt5 = connection.prepareStatement(sheet_mnemonic_update);
|
||||
stmt5.setInt(1, max_file_syspk);
|
||||
stmt5.executeUpdate();
|
||||
// System.out.println("SQL"+sheet_mnemonic_update+"stmt5:"+stmt5);
|
||||
|
||||
String sheet_mnemonic_update2 = " update staging1.staging_generic_table a set sheet_format_mnemonic = b.sheet_format,file_sheet_mnemonic=b.file_sheet_mnemonic"+
|
||||
" from fw_core.test_file_sheet_format_master b where a.file_syspk =? and a.file_mnemonic=b.file_mnemonic and split_part(lower(a.sheet_name),'-',1) = lower(b.sheet_name) and a.file_sheet_mnemonic is null;";
|
||||
PreparedStatement stmt6 = connection.prepareStatement(sheet_mnemonic_update2);
|
||||
stmt6.setInt(1, max_file_syspk);
|
||||
stmt6.executeUpdate();
|
||||
|
||||
String sheet_mnemonic_update3 = " update staging1.staging_generic_table a set sheet_format_mnemonic = b.sheet_format,file_sheet_mnemonic=b.file_sheet_mnemonic"+
|
||||
" from fw_core.test_file_sheet_format_master b where a.file_syspk =? and a.file_mnemonic='BUDNI' and split_part(lower(a.sheet_name),' ',1) = split_part(lower(b.sheet_name),' ',1) ;";
|
||||
PreparedStatement stmt7 = connection.prepareStatement(sheet_mnemonic_update3);
|
||||
stmt7.setInt(1, max_file_syspk);
|
||||
stmt7.executeUpdate();
|
||||
|
||||
connection.commit();
|
||||
|
||||
}catch (Exception e) {
|
||||
|
||||
System.out.println("Error reading file");
|
||||
System.out.print( "Exception:"+e+"\n");
|
||||
LOGGER.log(Level.SEVERE, "Exception occur", e);
|
||||
// String err_msg = "file_syspk-"+ max_file_syspk +":"+ ExceptionUtils.getStackTrace(e);
|
||||
|
||||
String err_msg = "file_syspk-"+ max_file_syspk +":"+ e.toString();
|
||||
|
||||
//inserting error message
|
||||
/* String sql_error_msg = " INSERT INTO fw_core.fw_log_app_errorlog (client_id, function_id, error_msg, create_timestamp, update_timestamp)\n" +
|
||||
"VALUES(4, 4, ?, now(),now())";
|
||||
PreparedStatement stmt_err_msg = connection2.prepareStatement(sql_error_msg);
|
||||
stmt_err_msg.setString(1,err_msg);
|
||||
stmt_err_msg.executeUpdate();*/
|
||||
//NA updates
|
||||
CallableStatement errorlogFunc = connection2.prepareCall("{ call fw_insert_app_error(?,?,null,?,null,null,null) }");
|
||||
errorlogFunc.setInt(1, client_id);
|
||||
errorlogFunc.setInt(2, function_id);
|
||||
errorlogFunc.setString(3, err_msg);
|
||||
errorlogFunc.execute();
|
||||
errorlogFunc.close();
|
||||
|
||||
/* String sql_sheet_level2 = " update fw_core.fw_jobctl_file_sheet_runschedule set end_status='error',end_status_note='stg1_failed' , end_time=now() where file_syspk=? ";
|
||||
PreparedStatement stmt8 = connection.prepareStatement(sql_sheet_level2);
|
||||
stmt8.setInt(1, max_file_syspk);
|
||||
stmt8.executeUpdate();*/
|
||||
|
||||
String sql_file_level3 = " update fw_core.fw_jobctl_file_runschedule set end_status='error',end_status_note='stg1_failed' , end_time=now() where file_syspk=? ";
|
||||
PreparedStatement stmt9 = connection.prepareStatement(sql_file_level3);
|
||||
stmt9.setInt(1, max_file_syspk);
|
||||
stmt9.executeUpdate();
|
||||
|
||||
String runsch_update = " update fw_core.fw_jobctl_runschedule set end_status='error',end_status_note='stg1_failed', end_time=now() where latest_runschedule_flag='1'; ";
|
||||
Statement stmt10 = connection.createStatement();
|
||||
stmt10.executeUpdate(runsch_update);
|
||||
|
||||
//update end_status in fw_jobctl_runschedule_jobstep
|
||||
String sql_file_level4 = " update fw_core.fw_jobctl_runschedule_jobstep set end_status='error',end_status_note='stg1_failed', end_time=now() where latest_runschedule_flag='1' and run_schedule_id=? and job_id=? and step_id=?;";
|
||||
PreparedStatement stmt11 = connection.prepareStatement(sql_file_level4);
|
||||
stmt11.setInt(1, max_run_schedule_id);
|
||||
stmt11.setInt(2, job_id);
|
||||
stmt11.setInt(3, step_id);
|
||||
stmt11.executeUpdate();
|
||||
|
||||
|
||||
continue;
|
||||
}
|
||||
}
|
||||
//update sheet_mnemonic in fw_jobctl_file_sheet_runschedule
|
||||
String sheet_mnemonic_update = " update fw_core.fw_jobctl_file_sheet_runschedule a set sheet_format_mnemonic = b.sheet_format,file_sheet_mnemonic=b.file_sheet_mnemonic\n" +
|
||||
"from fw_core.test_file_sheet_format_master b where a.file_syspk =? and a.file_mnemonic=b.file_mnemonic and trim(lower(a.sheet_name)) = lower(b.sheet_name) ;";
|
||||
PreparedStatement stmt5 = connection.prepareStatement(sheet_mnemonic_update);
|
||||
stmt5.setInt(1, max_file_syspk);
|
||||
stmt5.executeUpdate();
|
||||
// System.out.println("SQL"+sheet_mnemonic_update+"stmt5:"+stmt5);
|
||||
|
||||
String sheet_mnemonic_update2 = " update fw_core.fw_jobctl_file_sheet_runschedule a set sheet_format_mnemonic = b.sheet_format,file_sheet_mnemonic=b.file_sheet_mnemonic\n" +
|
||||
"from fw_core.test_file_sheet_format_master b where a.file_syspk =? and a.file_mnemonic=b.file_mnemonic and split_part(lower(a.sheet_name),'-',1) = lower(b.sheet_name) and a.file_sheet_mnemonic is null;";
|
||||
PreparedStatement stmt6 = connection.prepareStatement(sheet_mnemonic_update2);
|
||||
stmt6.setInt(1, max_file_syspk);
|
||||
stmt6.executeUpdate();
|
||||
|
||||
String sheet_mnemonic_update3 = " update fw_core.fw_jobctl_file_sheet_runschedule a set sheet_format_mnemonic = b.sheet_format,file_sheet_mnemonic=b.file_sheet_mnemonic\n" +
|
||||
"from fw_core.test_file_sheet_format_master b where a.file_syspk =? and a.file_mnemonic='BUDNI' and split_part(lower(a.sheet_name),' ',1) = split_part(lower(b.sheet_name),' ',1) ;";
|
||||
PreparedStatement stmt7 = connection.prepareStatement(sheet_mnemonic_update3);
|
||||
stmt7.setInt(1, max_file_syspk);
|
||||
stmt7.executeUpdate();
|
||||
|
||||
//update file count and end_status in fw_jobctl_runschedule
|
||||
String sql_file_level3 = " update fw_core.fw_jobctl_runschedule set end_status='success',end_status_note='stg1_completed', object_count=? , end_time=now() where latest_runschedule_flag='1' and run_schedule_id=? ;";
|
||||
PreparedStatement stmt9 = connection.prepareStatement(sql_file_level3);
|
||||
stmt9.setInt(1, file_count);
|
||||
stmt9.setInt(2, max_run_schedule_id);
|
||||
stmt9.executeUpdate();
|
||||
|
||||
//update end_status if file load is success in fw_jobctl_runschedule_jobstep
|
||||
String sql_file_level4 = " update fw_core.fw_jobctl_runschedule_jobstep set end_status='success',end_status_note='stg1_completed', end_time=now() where latest_runschedule_flag='1' and run_schedule_id=? and job_id=? and step_id=?;";
|
||||
PreparedStatement stmt11 = connection.prepareStatement(sql_file_level4);
|
||||
stmt11.setInt(1, max_run_schedule_id);
|
||||
stmt11.setInt(2, job_id);
|
||||
stmt11.setInt(3, step_id);
|
||||
stmt11.executeUpdate();
|
||||
|
||||
connection.commit();
|
||||
|
||||
|
||||
connection.close();
|
||||
connection2.close();
|
||||
|
||||
} catch (SQLException ex2) {
|
||||
System.out.println("Database error");
|
||||
ex2.printStackTrace();
|
||||
} catch (NullPointerException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
} else {
|
||||
System.out.println("There is no Excel file to process");
|
||||
String sql_file_level3 = " update fw_core.fw_jobctl_runschedule set end_status='error',end_status_note='no files in landing', end_time=now() where latest_runshcedule_flag=1 ";
|
||||
Statement st = null;
|
||||
try {
|
||||
st = connection.createStatement();
|
||||
st.executeUpdate(sql_file_level3);
|
||||
|
||||
} catch (SQLException throwables) {
|
||||
throwables.printStackTrace();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
public static class ExcelFileFilter implements java.io.FileFilter {
|
||||
public boolean accept(File file) {
|
||||
return file != null &&
|
||||
file.isFile() &&
|
||||
file.canRead() &&
|
||||
(file.getName().endsWith("xls")
|
||||
|| file.getName().endsWith("xlsx"));
|
||||
}
|
||||
}
|
||||
private static boolean isRowEmpty(Row row) {
|
||||
boolean isEmpty = true;
|
||||
DataFormatter dataFormatter = new DataFormatter();
|
||||
|
||||
if (row != null) {
|
||||
for (Cell cell : row) {
|
||||
if (dataFormatter.formatCellValue(cell).trim().length() > 0) {
|
||||
isEmpty = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return isEmpty;
|
||||
}
|
||||
|
||||
/*String getMergedRegionStringValue(Sheet sheet, int firstRow, int firstColumn){
|
||||
for(int i = 0; i < sheet.getNumMergedRegions(); i++) {
|
||||
CellRangeAddress region = sheet.getMergedRegion(i);
|
||||
|
||||
int colIndex = region.getFirstColumn();
|
||||
int rowNum = region.getFirstRow();
|
||||
//check first cell of the region
|
||||
if(rowNum == firstRow && colIndex == firstColumn){
|
||||
return sheet.getRow(rowNum).getCell(colIndex).getStringCellValue();
|
||||
}
|
||||
}
|
||||
}*/
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
Manifest-Version: 1.0
|
||||
Main-Class: org.example.App
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
package org.example;
|
||||
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
/**
|
||||
* Unit test for simple App.
|
||||
*/
|
||||
public class AppTest
|
||||
{
|
||||
/**
|
||||
* Rigorous Test :-)
|
||||
*/
|
||||
@Test
|
||||
public void shouldAnswerWithTrue()
|
||||
{
|
||||
assertTrue( true );
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
Manifest-Version: 1.0
|
||||
Main-Class: org.example.App
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,5 @@
|
||||
#Generated by Maven
|
||||
#Thu Jun 10 12:07:03 UTC 2021
|
||||
version=1.0-SNAPSHOT
|
||||
groupId=org.example
|
||||
artifactId=ex1
|
||||
@@ -0,0 +1,4 @@
|
||||
org/example/App$ExcelFileFilter.class
|
||||
org/example/example.class
|
||||
org/example/App$1.class
|
||||
org/example/App.class
|
||||
@@ -0,0 +1,2 @@
|
||||
/home/compegence/mmt_code/generic_table/src/main/java/org/example/App.java
|
||||
/home/compegence/mmt_code/generic_table/src/main/java/example.java
|
||||
@@ -0,0 +1 @@
|
||||
org/example/AppTest.class
|
||||
@@ -0,0 +1 @@
|
||||
/home/compegence/customer/MMT/java/generic_table/src/test/java/org/example/AppTest.java
|
||||
@@ -0,0 +1,64 @@
|
||||
<?xml version="1.0" encoding="UTF-8" ?>
|
||||
<testsuite tests="1" failures="0" name="org.example.AppTest" time="0.002" errors="0" skipped="0">
|
||||
<properties>
|
||||
<property name="java.runtime.name" value="OpenJDK Runtime Environment"/>
|
||||
<property name="sun.boot.library.path" value="/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64"/>
|
||||
<property name="java.vm.version" value="25.292-b10"/>
|
||||
<property name="java.vm.vendor" value="Private Build"/>
|
||||
<property name="maven.multiModuleProjectDirectory" value="/home/compegence/customer/MMT/java/generic_table"/>
|
||||
<property name="java.vendor.url" value="http://java.oracle.com/"/>
|
||||
<property name="path.separator" value=":"/>
|
||||
<property name="guice.disable.misplaced.annotation.check" value="true"/>
|
||||
<property name="java.vm.name" value="OpenJDK 64-Bit Server VM"/>
|
||||
<property name="file.encoding.pkg" value="sun.io"/>
|
||||
<property name="sun.java.launcher" value="SUN_STANDARD"/>
|
||||
<property name="sun.os.patch.level" value="unknown"/>
|
||||
<property name="java.vm.specification.name" value="Java Virtual Machine Specification"/>
|
||||
<property name="user.dir" value="/home/compegence/customer/MMT/java/generic_table"/>
|
||||
<property name="java.runtime.version" value="1.8.0_292-8u292-b10-0ubuntu1~18.04-b10"/>
|
||||
<property name="java.awt.graphicsenv" value="sun.awt.X11GraphicsEnvironment"/>
|
||||
<property name="java.endorsed.dirs" value="/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/endorsed"/>
|
||||
<property name="os.arch" value="amd64"/>
|
||||
<property name="java.io.tmpdir" value="/tmp"/>
|
||||
<property name="line.separator" value="
|
||||
"/>
|
||||
<property name="java.vm.specification.vendor" value="Oracle Corporation"/>
|
||||
<property name="os.name" value="Linux"/>
|
||||
<property name="classworlds.conf" value="/usr/share/maven/bin/m2.conf"/>
|
||||
<property name="sun.jnu.encoding" value="UTF-8"/>
|
||||
<property name="java.library.path" value="/usr/java/packages/lib/amd64:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib"/>
|
||||
<property name="maven.conf" value="/usr/share/maven/conf"/>
|
||||
<property name="java.specification.name" value="Java Platform API Specification"/>
|
||||
<property name="java.class.version" value="52.0"/>
|
||||
<property name="sun.management.compiler" value="HotSpot 64-Bit Tiered Compilers"/>
|
||||
<property name="os.version" value="4.15.0-65-generic"/>
|
||||
<property name="library.jansi.path" value="/usr/share/maven/lib/jansi-native"/>
|
||||
<property name="user.home" value="/home/compegence"/>
|
||||
<property name="user.timezone" value="Etc/UTC"/>
|
||||
<property name="java.awt.printerjob" value="sun.print.PSPrinterJob"/>
|
||||
<property name="java.specification.version" value="1.8"/>
|
||||
<property name="file.encoding" value="UTF-8"/>
|
||||
<property name="user.name" value="compegence"/>
|
||||
<property name="java.class.path" value="/usr/share/maven/boot/plexus-classworlds-2.x.jar"/>
|
||||
<property name="java.vm.specification.version" value="1.8"/>
|
||||
<property name="sun.arch.data.model" value="64"/>
|
||||
<property name="java.home" value="/usr/lib/jvm/java-8-openjdk-amd64/jre"/>
|
||||
<property name="sun.java.command" value="org.codehaus.plexus.classworlds.launcher.Launcher compile package shade:shade"/>
|
||||
<property name="java.specification.vendor" value="Oracle Corporation"/>
|
||||
<property name="user.language" value="en"/>
|
||||
<property name="awt.toolkit" value="sun.awt.X11.XToolkit"/>
|
||||
<property name="java.vm.info" value="mixed mode"/>
|
||||
<property name="java.version" value="1.8.0_292"/>
|
||||
<property name="java.ext.dirs" value="/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/ext:/usr/java/packages/lib/ext"/>
|
||||
<property name="securerandom.source" value="file:/dev/./urandom"/>
|
||||
<property name="sun.boot.class.path" value="/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/resources.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/rt.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/sunrsasign.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jsse.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jce.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/charsets.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jfr.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/classes"/>
|
||||
<property name="java.vendor" value="Private Build"/>
|
||||
<property name="maven.home" value="/usr/share/maven"/>
|
||||
<property name="file.separator" value="/"/>
|
||||
<property name="java.vendor.url.bug" value="http://bugreport.sun.com/bugreport/"/>
|
||||
<property name="sun.cpu.endian" value="little"/>
|
||||
<property name="sun.io.unicode.encoding" value="UnicodeLittle"/>
|
||||
<property name="sun.cpu.isalist" value=""/>
|
||||
</properties>
|
||||
<testcase classname="org.example.AppTest" name="shouldAnswerWithTrue" time="0.002"/>
|
||||
</testsuite>
|
||||
@@ -0,0 +1,4 @@
|
||||
-------------------------------------------------------------------------------
|
||||
Test set: org.example.AppTest
|
||||
-------------------------------------------------------------------------------
|
||||
Tests run: 1, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.061 sec
|
||||
Binary file not shown.
97
MMT_latest_version/java/load.sh
Normal file
97
MMT_latest_version/java/load.sh
Normal file
@@ -0,0 +1,97 @@
|
||||
#!/bin/bash -e
|
||||
|
||||
#!/bin/bash -e
|
||||
HOME="/home/compegence/customer"
|
||||
#Directories
|
||||
homeDir=$HOME/MMT
|
||||
configDir=$homeDir/conf
|
||||
input_file=$configDir/etl-config.txt
|
||||
declare -A credentials
|
||||
while IFS="=" read -r key value; do
|
||||
credentials[$key]=$value
|
||||
done < <( sed -e '/^\s*$/ d' -e '/^#/ d' $input_file )
|
||||
for i in ${credentials[@]}; do echo ***$i; done
|
||||
|
||||
user=${credentials[user]}
|
||||
password=${credentials[password]}
|
||||
host=${credentials[host]}
|
||||
#db=${credentials[db]}
|
||||
protocol=${credentials[protocol]}
|
||||
server=${credentials[server]}
|
||||
db="m_test"
|
||||
|
||||
mmt_con="psql -X -A -t postgresql://$user:$password@$host/$db?options=--search_path%3D"
|
||||
|
||||
#let i=0
|
||||
#set -A arr
|
||||
#BASE_CODS_ARGUMENTS="--host=$cods_host --user=$cods_user --password=$cods_pwd --port=$3306"
|
||||
#BASE_ODS_ARGUMENTS="--user=$user --password=$pwd --host=$host"
|
||||
psql_base_arg="postgresql://$user:$password@$host/$db"
|
||||
#psql_base_arg="postgresql://postgres:j3(jLBq}e@localhost/ffm_k2"
|
||||
echo $psql_base_arg
|
||||
|
||||
ifStart=`date '+%d'`
|
||||
echo "$ifStart"
|
||||
|
||||
|
||||
#Check today is sunday. Scheduling weekly job
|
||||
today="$(date +%a)"
|
||||
sun="Mon"
|
||||
|
||||
psql $psql_base_arg -c "update fw_core.fw_jobctl_runschedule set latest_runschedule_flag='0' where latest_runschedule_flag='1';"
|
||||
psql $psql_base_arg -c "update fw_core.fw_jobctl_runschedule_jobstep set latest_runschedule_flag='0' where latest_runschedule_flag='1';"
|
||||
|
||||
|
||||
#inserting into fw_core.fw_jobctl_runschedule
|
||||
chkrun=$(psql $psql_base_arg -t -c "select count(*) from fw_core.fw_jobctl_jobstep_master where client_id=21 and function_id=1 and run_frequency='daily' and active_flag=true;")
|
||||
echo "chkrun is $chkrun"
|
||||
if [[ $chkrun -gt 0 ]]; then
|
||||
echo "continuing";
|
||||
psql $psql_base_arg -c "INSERT INTO fw_core.fw_jobctl_runschedule
|
||||
(client_id, function_id, run_schedule_timestamp, run_frequency, start_time, begin_status, created_by, updated_by, create_timestamp, update_timestamp)
|
||||
select client_id, function_id,now(),run_frequency,now(),'','Admin','Admin',now(),now() from fw_core.fw_jobctl_jobstep_master
|
||||
where active_flag='TRUE' group by client_id, function_id, run_frequency limit 1;"
|
||||
|
||||
#check previous day job status in fw_jobctl_runschedule
|
||||
chkrun2=$(psql $psql_base_arg -t -c "select trim(end_status) from fw_core.fw_jobctl_runschedule where run_schedule_id = (
|
||||
select max(run_schedule_id) as max from fw_core.fw_jobctl_runschedule where latest_runschedule_flag='0' and client_id=21 and function_id=1);")
|
||||
echo "chkrun2 is $chkrun2"
|
||||
|
||||
if [ "$chkrun2" != " error" ]; then
|
||||
echo "welcome1"
|
||||
|
||||
#Update fw_jobctl_runschedule
|
||||
psql $psql_base_arg -c "update fw_core.fw_jobctl_runschedule set begin_status='started' where client_id=21 and function_id=1 and run_frequency='daily' and latest_runschedule_flag='1' "
|
||||
else
|
||||
psql $psql_base_arg -c "update fw_core.fw_jobctl_runschedule set begin_status='started',end_status='error',end_status_note = 'previous day job failed' where client_id=21 and function_id=1 and run_frequency='daily' and latest_runschedule_flag='1' "
|
||||
echo "welcome2"
|
||||
exit 1;
|
||||
fi
|
||||
else
|
||||
echo " No active job for daily for client_id=21"
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
# looping fw_jobctl_jobstep_master for active records
|
||||
while IFS="|" read -a line
|
||||
do
|
||||
run_frequency=${line[2]}
|
||||
job_scheduling_day="${line[1]}"
|
||||
|
||||
echo "$run_frequency"
|
||||
if [ $run_frequency == "daily" ] ; then
|
||||
echo "Populating daily job into fw_jobctl_runschedule_jobstep"
|
||||
|
||||
psql $psql_base_arg -c"insert into fw_core.fw_jobctl_runschedule_jobstep(client_id, function_id,latest_runschedule_flag,run_schedule_id,run_schedule_timestamp, run_frequency, job_id, step_id, job_step_run_dependency_seuqence,
|
||||
job_script_type,job_step_script_name,job_fun_param_array,job_name, step_name, job_scope, job_scope_qualifier_array, created_by,updated_by, create_timestamp, update_timestamp)
|
||||
select a.client_id, a.function_id,b.latest_runschedule_flag, b.run_schedule_id,b.run_schedule_timestamp,a.run_frequency, a.job_id, a.step_id, a.job_step_run_dependency_seuqence,
|
||||
a.job_script_type,a.job_step_script_name,a.job_fun_param_array,a.job_name,a.step_name,a.job_scope, a.job_scope_qualifier_array, 'Admin','Admin',now(),now()
|
||||
from fw_core.fw_jobctl_jobstep_master a join fw_core.fw_jobctl_runschedule b
|
||||
on a.client_id=b.client_id and a.function_id=b.function_id and a.run_frequency=b.run_frequency
|
||||
where latest_runschedule_flag='1' and b.begin_status = 'started' and b.end_status is null and a.active_flag='true' and a.run_frequency='daily'
|
||||
order by a.client_id,a.function_id,a.job_id,a.step_id,a.job_step_run_dependency_seuqence;"
|
||||
|
||||
fi
|
||||
done < <(psql $psql_base_arg -At -c"select distinct client_id,function_id,run_frequency from fw_core.fw_jobctl_jobstep_master order by run_frequency ")
|
||||
|
||||
|
||||
127
MMT_latest_version/java/run.sh
Normal file
127
MMT_latest_version/java/run.sh
Normal file
@@ -0,0 +1,127 @@
|
||||
#!/bin/bash -e
|
||||
HOME="/home/compegence/customer"
|
||||
#Directories
|
||||
homeDir=$HOME/MMT
|
||||
configDir=$homeDir/conf
|
||||
input_file=$configDir/etl-config.txt
|
||||
declare -A credentials
|
||||
while IFS="=" read -r key value; do
|
||||
credentials[$key]=$value
|
||||
done < <( sed -e '/^\s*$/ d' -e '/^#/ d' $input_file )
|
||||
for i in ${credentials[@]}; do echo ***$i; done
|
||||
|
||||
user=${credentials[user]}
|
||||
password=${credentials[password]}
|
||||
host=${credentials[host]}
|
||||
db=${credentials[db]}
|
||||
protocol=${credentials[protocol]}
|
||||
server=${credentials[server]}
|
||||
db="m_test"
|
||||
|
||||
#mmt_con="psql -X -A -t postgresql://$user:$password@$host/$db?options=--search_path%3D"
|
||||
|
||||
psql_base_arg="postgresql://$user:$password@$host/$db"
|
||||
echo $psql_base_arg
|
||||
|
||||
# load the record into the fw_jobctl_runschedule accounting for run frequency (hourly, daily, monthly, yearly) and insert
|
||||
## ???? Should this go into the first loop; to load only one runschedule
|
||||
#/home/compegence/customer/MMT/job_schedule/job_load.sh
|
||||
|
||||
##***** Run frequency loop; Outer Loop
|
||||
## ???? Should we change it to select runschedules applicable, from the fw_jobctl_jobstep_master; and based on that run the loop
|
||||
#while loop for each client_id,function_id,run_frequency from fw_jobctl_runschedule_jobstep
|
||||
|
||||
while IFS="|" read -a outer
|
||||
do
|
||||
client_id=${outer[0]}
|
||||
function_id=${outer[1]}
|
||||
run_frequency=${outer[2]}
|
||||
|
||||
#Update start_time for fw_jobctl_runschedule
|
||||
#psql $psql_base_arg -c "update fw_core.fw_jobctl_runschedule set start_time=now(),begin_status='started' where latest_runschedule_flag='1' and client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' and end_status is null "
|
||||
|
||||
##** Job and Step Loop; innner loop;
|
||||
#while loop for each job & step
|
||||
##** Job and Step Loop; innner loop; based on the entries on the table fw_jobctl_runschedule_jobstep
|
||||
|
||||
job_status=0
|
||||
prev_job_seq=''
|
||||
while IFS="|" read -a inner
|
||||
do
|
||||
client_id=${inner[0]}
|
||||
function_id=${inner[1]}
|
||||
run_frequency=${inner[2]}
|
||||
job_name=${inner[4]}
|
||||
step_name+=(${inner[5]})
|
||||
job_script_type=${inner[6]}
|
||||
job_step_script_name=${inner[7]}
|
||||
#remove first and last char - curly braces
|
||||
job_fun_param_array=${inner[8]:1:-1}
|
||||
job_step_run_dependency_seuqence=${inner[9]}
|
||||
|
||||
param=''
|
||||
var1=' '
|
||||
|
||||
#reading array column and spliting each var and appending $ & space
|
||||
IFS=',' read -r -a ADDR <<< "$job_fun_param"
|
||||
for i in "${ADDR[@]}"; do
|
||||
echo $i
|
||||
param=$param$i$var1
|
||||
done
|
||||
echo $param
|
||||
|
||||
#previous job_step_run_dependency_seuqence
|
||||
#prev_job_seq=$job_step_run_dependency_seuqence
|
||||
echo $prev_job_seq
|
||||
## Process java Program
|
||||
if [[ $job_script_type == "java" ]]; then
|
||||
echo " $job_step_script_name started"
|
||||
psql $psql_base_arg -c "update fw_core.fw_jobctl_runschedule_jobstep set start_time=now(),begin_status='started' where job_step_script_name='$job_step_script_name' and client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' and latest_runschedule_flag='1' "
|
||||
|
||||
#java -jar /home/compegence/mmt_code/generic_table/target/ex1-1.0-SNAPSHOT-shaded.jar
|
||||
java -jar $job_step_script_name
|
||||
echo " $job_step_script_name Ended"
|
||||
|
||||
## Process sql files
|
||||
elif [[ $job_script_type == "sql" ]] ; then
|
||||
|
||||
chkrun=$(psql $psql_base_arg -t -c "select TRIM(end_status) from fw_core.fw_jobctl_runschedule_jobstep where job_step_run_dependency_seuqence=$prev_job_seq and client_id=21 and function_id=1;")
|
||||
echo "chkrun is $chkrun"
|
||||
|
||||
if [[ "$chkrun" == " success" ]]; then
|
||||
echo "job_function_name started"
|
||||
psql $psql_base_arg -a -f $job_step_script_name $host $port $user $pwd $dbname $param $client_id $function_id $data_from_date $data_to_date $job_scope $job_scope_qualifier
|
||||
|
||||
else
|
||||
echo "exit"
|
||||
exit 1;
|
||||
fi
|
||||
echo " $job_step_script_name Ended"
|
||||
|
||||
## Process java Program
|
||||
elif [[ $job_script_type == "shell_script" ]] ; then
|
||||
chkrun=$(psql $psql_base_arg -t -c "select TRIM(end_status) from fw_core.fw_jobctl_runschedule_jobstep where job_step_run_dependency_seuqence=$prev_job_seq and client_id=21 and function_id=1;")
|
||||
echo "chkrun is $chkrun"
|
||||
|
||||
if [ "$chkrun" == " success" ]; then
|
||||
echo "$job_step_script_name started"
|
||||
$job_step_script_name
|
||||
echo "$job_step_script_name ended"
|
||||
else
|
||||
exit 1;
|
||||
fi
|
||||
fi
|
||||
|
||||
prev_job_seq=$job_step_run_dependency_seuqence
|
||||
echo $prev_job_seq
|
||||
## End of While Loop
|
||||
#2nd looping through fw_core.fw_jobctl_runschedule_jobstep for each client_id,function_id,run_frequency,run_schedule_id1 is completed
|
||||
done < <(psql $psql_base_arg -At -c"select client_id,function_id,run_frequency,run_schedule_id,job_name,step_name,job_script_type,job_step_script_name,job_fun_param_array,job_step_run_dependency_seuqence from fw_core.fw_jobctl_runschedule_jobstep where latest_runschedule_flag='1' and client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' order by run_schedule_id,job_step_run_dependency_seuqence; ")
|
||||
|
||||
|
||||
#Update end_time for fw_jobctl_runschedule
|
||||
psql $psql_base_arg -c "update fw_core.fw_jobctl_runschedule set latest_runschedule_flag='0',end_time=now() where client_id=$client_id and function_id=$function_id and run_frequency='$run_frequency' and latest_runschedule_flag='1' and end_status is null "
|
||||
#1st loop for client_id,function_id,run_frequency
|
||||
done < <(psql $psql_base_arg -At -c"select distinct client_id,function_id,run_frequency from fw_core.fw_jobctl_runschedule_jobstep where latest_runschedule_flag='1' order by client_id,function_id,run_frequency ")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user