package org.example;

import java.io.*;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.sql.*;
import java.text.SimpleDateFormat;
import java.util.*;

import org.apache.poi.openxml4j.util.ZipSecureFile;
import org.apache.poi.ss.usermodel.*;
import org.apache.poi.ss.usermodel.Row.MissingCellPolicy;
import org.apache.poi.ss.util.CellRangeAddress;
import org.apache.poi.xssf.usermodel.*;

import java.io.FileInputStream;
import java.io.FileNotFoundException;

import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.FormulaEvaluator;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.xmlbeans.impl.values.XmlValueDisconnectedException;

import java.nio.file.*;
import java.nio.file.attribute.*;
import java.util.logging.Logger;
import java.util.logging.Level;

public class App {
    private String sourceFile;
    private String outputFile;
    private XSSFWorkbook workbook;
    private static XSSFCell Cell;
    private static XSSFRow Row;
    private static MissingCellPolicy xRow;

    private final static Logger LOGGER =
            Logger.getLogger(Logger.GLOBAL_LOGGER_NAME);

    public static void main(String[] args) throws SQLException {

        // private static Logger logger = Logger.getLogger(MyClass.class);
        FileFilter filter = new ExcelFileFilter();

        //File src_directory = new File("D:\\customer\\MMT\\files\\landing");
        File src_directory = new File("/home/compegence/customer/MMT/files/landing");

       // File destination = new File("D:\\customer\\MMT\\files\\processed");
        File destination = new File("/home/compegence/customer/MMT/files/processed");

        //Exception:org.apache.poi.ooxml.POIXMLException: Zip bomb detected!
        ZipSecureFile.setMinInflateRatio(0);

        String db = "", host = "", username = "", password = "", port = "";
        String[] split_arr = new String[100];
        String[] codes = new String[100];
        int c = 0;
        //  String HomeDir =  System.getProperty("user.home");
        Scanner input = null;

        App app = new App();

        try {
            input = new Scanner(new File("/home/compegence/customer/MMT/conf/etl-config.txt"));
           // input = new Scanner(new File("etl-config.txt"));
            while (input.hasNext()) {
                codes[c] = input.nextLine();

                if (codes[c].contains("db")) {
                    split_arr = codes[c].split("=");
                    db = split_arr[1];
                }
                if (codes[c].contains("host")) {
                    split_arr = codes[c].split("=");
                    host = split_arr[1];
                }
                if (codes[c].contains("user")) {
                    split_arr = codes[c].split("=");
                    username = split_arr[1];
                }
                if (codes[c].contains("password")) {
                    split_arr = codes[c].split("=");
                    password = split_arr[1];
                }
                if (codes[c].contains("port")) {
                    split_arr = codes[c].split("=");
                    port = split_arr[1];
                }
                c++;
            }

        } catch (FileNotFoundException e) {
            e.printStackTrace();
        }
        //db="m_test";
        //String jdbcURL = "jdbc:postgresql://localhost:2345/m_test?currentSchema=staging1";
        String jdbcURL = "jdbc:postgresql://" + host + ":" + port + "/" + db + "?currentSchema=staging1";

        Connection connection = null;
        Connection con = null;
        Statement stmt = null;

        String jdbcURL2 = "jdbc:postgresql://localhost:2345/fw_demo?currentSchema=fw_core";
        Connection connection2 = null;

        int client_id = 0, function_id = 0, job_id = 0, step_id = 0, max_run_schedule_id = 0;
        DataFormatter formatter = new DataFormatter(true);

        if (src_directory.isDirectory()) {
            File[] files = src_directory.listFiles(filter);
            // if (src_directory.length() >= 0) {
            if (files.length > 0) {
                try {
                    connection2 = DriverManager.getConnection(jdbcURL2, username, password);
                    connection2.setAutoCommit(false);

                    //set connection for mmt
                    connection = DriverManager.getConnection(jdbcURL, username, password);
                    connection.setAutoCommit(false);

                    java.sql.Timestamp run_schedule_date = java.sql.Timestamp.valueOf("1900-01-01 00:00:00");

                    //Pick up the client_id, function_id and run_schedule_id
                    stmt = connection.createStatement();
                    String qry0 = "select client_id,function_id,run_schedule_id,run_schedule_timestamp,job_id,step_id from fw_core.fw_jobctl_runschedule_jobstep where latest_runschedule_flag='1' and job_script_type='java' ;";
                    ResultSet rs0 = stmt.executeQuery(qry0);
                    while (rs0.next()) {
                        client_id = rs0.getInt("client_id");
                        function_id = rs0.getInt("function_id");
                        max_run_schedule_id = rs0.getInt("run_schedule_id");
                        run_schedule_date = rs0.getTimestamp("run_schedule_timestamp");
                        job_id = rs0.getInt("job_id");
                        step_id = rs0.getInt("step_id");
                    }
                    rs0.close();

                    //  java.sql.Timestamp file_created_date1 = new java.sql.Timestamp(run_schedule_date.getTime());

                    int max_file_syspk = 1;
                    int file_count = 0;

                    for (File file : files) {
                        try {
                            System.out.println("run_schedule_date: " + run_schedule_date);

                            file_count++;
                            Path path = Paths.get(file.getPath());
                            BasicFileAttributes attr;
                            //init workbook and do stuff

                            //Extracting 2nd word from file name
                            String[] file_name_split_array = file.getName().split("_", 3);
                            String file_format = file_name_split_array[1];
                            // String file_name = file_name_split_array[2];
                            String file_name = file.getName();

                            //String to date conversion
                            String startDate = file_name_split_array[0];//"01-02-2013";
                            SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
                            java.util.Date date = sdf.parse(startDate);
                            java.sql.Date fileDate = new java.sql.Date(date.getTime());

                            //Reading file creation time from directory
                            attr = Files.readAttributes(path, BasicFileAttributes.class);
                            FileTime fileTime = attr.creationTime();
                            java.util.Date utilDate = new java.util.Date(fileTime.toMillis());
                            java.sql.Timestamp file_created_date = new java.sql.Timestamp(utilDate.getTime());

                            //Get max file_syspk from mmt_ods.fw_jobctl_file_runschedule
                            stmt = connection.createStatement();
                            String qry1 = "select max(file_syspk) as max_file_syspk from  fw_core.fw_jobctl_file_runschedule;";
                            ResultSet rs1 = stmt.executeQuery(qry1);
                            while (rs1.next()) {
                                max_file_syspk = rs1.getInt("max_file_syspk");
                            }
                            rs1.close();
                            max_file_syspk++;

                            //insert record for each file in fw_jobctl_file_runschedule
                            String sql_file_level = "  INSERT INTO fw_core.fw_jobctl_file_runschedule(client_id, function_id,latest_runschedule_flag,run_schedule_id,job_id,step_id, file_syspk,file_name, file_mnemonic,  file_landing_date,staging_type, begin_status,  start_time,  created_by, updated_by, create_timestamp, update_timestamp)" +
                                    "VALUES(?,?,'1',?,?,?,?,?,?,?,?,'Started',now(), 'Admin','Admin',now(), now());";
                            PreparedStatement stmt1 = connection.prepareStatement(sql_file_level);

                            stmt1.setInt(1, client_id);
                            stmt1.setInt(2, function_id);
                            stmt1.setInt(3, max_run_schedule_id);
                            stmt1.setInt(4, job_id);
                            stmt1.setInt(5, step_id);
                            stmt1.setInt(6, max_file_syspk);
                            stmt1.setString(7, file_name);
                            stmt1.setString(8, file_format);
                            stmt1.setTimestamp(9, file_created_date);
                            stmt1.setString(10, "stg1");

                            stmt1.executeUpdate();

                            System.out.println("max_file_syspk: " + max_file_syspk);
                            long start = System.currentTimeMillis();

                            FileInputStream inputStream = new FileInputStream(file);

                            Workbook workbook = new XSSFWorkbook(inputStream);
                            // Workbook workbook = WorkbookFactory.create(file);

                            for (int k = 0; k < workbook.getNumberOfSheets(); k++) {
                                try {
                                    Sheet sheet1 = workbook.getSheetAt(k);
                                    String sheetName = workbook.getSheetAt(k).getSheetName();

                                    //  if (file_format.equals("FTDRY") &  ( ! sheetName.equals("Tractor specifications") | ! sheetName.equals("Summary"))){
                   /*     if (file_format.equals("FTDRY") &  ( (! sheetName.equals("Tractor specifications")) | (! sheetName.equals("Summary")))){

                        //System.out.println("sheet name summary:" + sheet1.getSheetName());
                          //  break;
                            k++;
                        }*/
                                    //insert record for each sheet of a file in fw_jobctl_file_sheet_runschedule
                                    String sql_sheet_level = "  INSERT INTO fw_core.fw_jobctl_file_sheet_runschedule(client_id, function_id,latest_runschedule_flag, run_schedule_id,job_id, step_id," +
                                            " file_syspk,  sheet_id,sheet_name, file_mnemonic,staging_type, begin_status,  start_time,  created_by, updated_by, create_timestamp, update_timestamp)" +
                                            "VALUES(?,?,'1',?,?,?,?,?,?,?,?,'Started',now(), 'Admin','Admin',now(), now());";
                                    PreparedStatement stmt3 = connection.prepareStatement(sql_sheet_level);

                                    stmt3.setInt(1, client_id);
                                    stmt3.setInt(2, function_id);
                                    stmt3.setInt(3, max_run_schedule_id);
                                    stmt3.setInt(4, job_id);
                                    stmt3.setInt(5, step_id);
                                    stmt3.setInt(6, max_file_syspk);
                                    stmt3.setInt(7, k);
                                    stmt3.setString(8, sheetName);
                                    stmt3.setString(9, file_format);
                                    stmt3.setString(10, "stg1");


                                    stmt3.executeUpdate();

                                    String[] column = new String[55];//new String[sheet1.getRow(1).getPhysicalNumberOfCells() + 1];
                                   // System.out.println("sheet name:" + sheet1.getSheetName());

                                    for (int i = 0; i <= sheet1.getLastRowNum(); i++) {
                                        Row row = sheet1.getRow(i);
                                        if (isRowEmpty(row)) {
                                            continue;
                                        }
                                        for (int j = 0; j < sheet1.getRow(i).getPhysicalNumberOfCells() + 1; j++) {
                                            Cell cell = row.getCell(j);
                                            if ((cell == null) || (cell.equals(""))) {
                                                continue;
                                            }

                                            //Removing array formula and setting modified formula. This error exists only in summary sheet
                                            if (cell.isPartOfArrayFormulaGroup()) {
                                                CellRangeAddress arrayRange = cell.getArrayFormulaRange();
                                                if (arrayRange.getNumberOfCells() > 1) {
                                                    String formula = row.getCell(j).getCellFormula();

                                                    sheet1.getRow(i).getSheet().removeArrayFormula(cell);
                                                    String[] formula1 = formula.split(":", 2);

                                                    row.getCell(j).setCellFormula(formula1[0]);
                                                   // System.out.println("Hello:" + formula1[0]);
                                                }
                                            }

                                            FormulaEvaluator formulaEval = workbook.getCreationHelper().createFormulaEvaluator();
                                            try {
                                                switch (row.getCell(j).getCellType()) {
                                                    case STRING:
                                                        // cell.setCellType(CellType.STRING);
                                                        row.getCell(j).setCellType(CellType.STRING);
                                                        // System.out.println("string: " + cell.getRichStringCellValue().getString());
                                                        break;
                                                    case NUMERIC:
                                                        //cell.setCellType(CellType.STRING);
                                                        row.getCell(j).setCellType(CellType.STRING);
                                                        //System.out.println("Numeric" + cell.getRichStringCellValue().getString());
                                                        break;
                                                    case BOOLEAN:
                                                        // cell.setCellType(CellType.STRING);
                                                        row.getCell(j).setCellType(CellType.STRING);
                                                        break;
                                                    case FORMULA:
                                                        row.getCell(j).setCellType(CellType.STRING);
                                                        // System.out.println("formula: " + cell.getRichStringCellValue().getString());
                                                        break;
                                                    case BLANK:
                                                        // row.getCell(j).setCellType(CellType.STRING);
                                                        break;
                                                    case _NONE:
                                                        //  row.getCell(j).setCellType(CellType.STRING);
                                                        break;
                                                    case ERROR:
                                                        // row.getCell(j).setCellType(CellType.STRING);
                                                        break;
                                                    default:
                                                        System.out.println("Default");
                                                }
                                            } catch (XmlValueDisconnectedException e) {
                                                String v = formatter.formatCellValue(sheet1.getRow(i).getCell(j), formulaEval);
                                                row.getCell(j).setCellValue(v);
                                                System.out.println("tested after:" + v);
                                            }

                                            column[j] = row.getCell(j).getStringCellValue();
                                        }

                                        //inserting each line of a sheet in staging_generic_table
                                        String sql = "INSERT INTO staging1.staging_generic_table (file_syspk,file_name,file_prefix_date,file_mnemonic,sheet_id,sheet_name,row_number,file_creation_date,column1, column2, column3,column4,column5,column6,column7,column8,column9,column10,column11,column12,column13,column14,column15,column16,column17,column18,column19,column20,column21,column22,column23,column24," +
                                                "column25,column26,column27,column28,column29,column30,column31,column32,column33,column34,column35,column36,column37,column38,column39,column40,column41,column42,column43,column44,column45,column46,column47,column48,column49,column50,column51,column52,column53)" +
                                                " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";

                                        PreparedStatement statement = connection.prepareStatement(sql);

                                        statement.setInt(1, max_file_syspk);
                                        statement.setString(2, file.getName());
                                        statement.setDate(3, fileDate);
                                        statement.setString(4, file_format);
                                        statement.setInt(5, k);
                                        statement.setString(6, sheetName);
                                        statement.setInt(7, i);
                                        statement.setTimestamp(8, file_created_date);
                                        statement.setString(9, column[0]);
                                        statement.setString(10, column[1]);
                                        statement.setString(11, column[2]);
                                        statement.setString(12, column[3]);
                                        statement.setString(13, column[4]);
                                        statement.setString(14, column[5]);
                                        statement.setString(15, column[6]);
                                        statement.setString(16, column[7]);
                                        statement.setString(17, column[8]);
                                        statement.setString(18, column[9]);
                                        statement.setString(19, column[10]);
                                        statement.setString(20, column[11]);
                                        statement.setString(21, column[12]);
                                        statement.setString(22, column[13]);
                                        statement.setString(23, column[14]);
                                        statement.setString(24, column[15]);
                                        statement.setString(25, column[16]);
                                        statement.setString(26, column[17]);
                                        statement.setString(27, column[18]);
                                        statement.setString(28, column[19]);
                                        statement.setString(29, column[20]);
                                        statement.setString(30, column[21]);
                                        statement.setString(31, column[22]);
                                        statement.setString(32, column[23]);
                                        statement.setString(33, column[24]);
                                        statement.setString(34, column[25]);
                                        statement.setString(35, column[26]);
                                        statement.setString(36, column[27]);
                                        statement.setString(37, column[28]);
                                        statement.setString(38, column[29]);
                                        statement.setString(39, column[30]);
                                        statement.setString(40, column[31]);
                                        statement.setString(41, column[32]);
                                        statement.setString(42, column[33]);
                                        statement.setString(43, column[34]);
                                        statement.setString(44, column[35]);
                                        statement.setString(45, column[36]);
                                        statement.setString(46, column[37]);
                                        statement.setString(47, column[38]);
                                        statement.setString(48, column[39]);
                                        statement.setString(49, column[40]);
                                        statement.setString(50, column[41]);
                                        statement.setString(51, column[42]);
                                        statement.setString(52, column[43]);
                                        statement.setString(53, column[44]);
                                        statement.setString(54, column[45]);
                                        statement.setString(55, column[46]);
                                        statement.setString(56, column[47]);
                                        statement.setString(57, column[48]);
                                        statement.setString(58, column[49]);
                                        statement.setString(59, column[50]);
                                        statement.setString(60, column[51]);
                                        statement.setString(61, column[52]);

                                        statement.executeUpdate();
                                    }

                                    //updating if sheet loaded successfully in fw_jobctl_file_sheet_runschedule
                                    String sql_sheet_level2 = "  update fw_core.fw_jobctl_file_sheet_runschedule set end_status='success',end_status_note='stg1_completed' , end_time=now() where file_syspk=? and sheet_id=? ";
                                    PreparedStatement stmt4 = connection.prepareStatement(sql_sheet_level2);

                                    stmt4.setInt(1, max_file_syspk);
                                    stmt4.setInt(2, k);
                                    stmt4.executeUpdate();

                                    connection.commit();

                                    long end = System.currentTimeMillis();
                                    System.out.printf("Import done in %d ms\n", (end - start));

                                } catch (Exception e) {
                                    String sheet_err_msg = "file_syspk-" + max_file_syspk + "sheet_id:" + k + ":" + e.toString();
                                    //inserting error message
                                    CallableStatement errorlogFunc = connection2.prepareCall("{ call fw_insert_app_error(?,?,null,?,null,null,null) }");
                                    errorlogFunc.setInt(1, client_id);
                                    errorlogFunc.setInt(2, function_id);
                                    errorlogFunc.setString(3, sheet_err_msg);
                                    errorlogFunc.execute();
                                    errorlogFunc.close();
                                    //updating sheet level failure
                                    String sql_sheet_level2 = "  update fw_core.fw_jobctl_file_sheet_runschedule set end_status='error',end_status_note='stg1_failed' , end_time=now() where file_syspk=? and sheet_id=? ";
                                    PreparedStatement stmt8 = connection.prepareStatement(sql_sheet_level2);
                                    stmt8.setInt(1, max_file_syspk);
                                    stmt8.setInt(2, k);
                                    stmt8.executeUpdate();

                                    continue;
                                }
                            }
                            workbook.close();

                            //Move the file after reading
                            inputStream.close(); //close before moving the file stream
                            Path source = Paths.get(src_directory.toPath() + "//" + file.getName());
                            Path target = Paths.get(destination.toPath() + "//" + file.getName());
                            //   java.nio.file.Files.move(Paths.get(src_directory.toPath()+"//"+file.getName()), Paths.get(destination.toPath()+"//"+file.getName()),StandardCopyOption.REPLACE_EXISTING);
                            //  java.nio.file.Files.move(source, target);

                            //updating if file loaded successfully in fw_jobctl_file_runschedule
                            //      String sql_file_level1 = "  update fw_core.fw_jobctl_file_runschedule set end_status='success',end_status_note='stg1_completed', end_time=now() where file_syspk=?  ";
                            String sql_file_level1 = "  update fw_core.fw_jobctl_file_runschedule a " +
                                    "set end_status = (case when t2.end_status='error' then 'error' when t2.end_status='success' then 'success' end)," +
                                    " end_status_note = case when t2.end_status='error' then 'stg1_failed' when t2.end_status='success' then 'stg1_completed' end " +
                                    "from  (select distinct file_syspk,end_status from fw_core.fw_jobctl_file_sheet_runschedule where file_syspk=?) t2 " +
                                    "where a.file_syspk = t2.file_syspk and a.run_schedule_id=?;";
                            PreparedStatement stmt2 = connection.prepareStatement(sql_file_level1);
                            stmt2.setInt(1, max_file_syspk);
                            stmt2.setInt(2, max_run_schedule_id);
                            stmt2.executeUpdate();

                            //NA updates
                            CallableStatement cleanFunc = connection.prepareCall("{ call fw_core.fn_update_na(?) }");
                            cleanFunc.setInt(1, max_file_syspk);
                            cleanFunc.execute();
                            cleanFunc.close();

                            //update sheet_mnemonic
                            String sheet_mnemonic_update = " update staging1.staging_generic_table a set sheet_format_mnemonic = b.sheet_format,file_sheet_mnemonic=b.file_sheet_mnemonic" +
                                    " from fw_core.test_file_sheet_format_master b where a.file_syspk =? and a.file_mnemonic=b.file_mnemonic and trim(lower(a.sheet_name)) = lower(b.sheet_name) ;";
                            PreparedStatement stmt5 = connection.prepareStatement(sheet_mnemonic_update);
                            stmt5.setInt(1, max_file_syspk);
                            stmt5.executeUpdate();
                            // System.out.println("SQL"+sheet_mnemonic_update+"stmt5:"+stmt5);

                            String sheet_mnemonic_update2 = " update staging1.staging_generic_table a set sheet_format_mnemonic = b.sheet_format,file_sheet_mnemonic=b.file_sheet_mnemonic" +
                                    " from fw_core.test_file_sheet_format_master b where a.file_syspk =? and a.file_mnemonic=b.file_mnemonic and split_part(lower(a.sheet_name),'-',1) = lower(b.sheet_name) and a.file_sheet_mnemonic is null;";
                            PreparedStatement stmt6 = connection.prepareStatement(sheet_mnemonic_update2);
                            stmt6.setInt(1, max_file_syspk);
                            stmt6.executeUpdate();

                            String sheet_mnemonic_update3 = " update staging1.staging_generic_table a set sheet_format_mnemonic = b.sheet_format,file_sheet_mnemonic=b.file_sheet_mnemonic" +
                                    " from fw_core.test_file_sheet_format_master b where a.file_syspk =? and a.file_mnemonic='BUDNI' and split_part(lower(a.sheet_name),' ',1) = split_part(lower(b.sheet_name),' ',1) ;";
                            PreparedStatement stmt7 = connection.prepareStatement(sheet_mnemonic_update3);
                            stmt7.setInt(1, max_file_syspk);
                            stmt7.executeUpdate();

                            connection.commit();

                        } catch (Exception e) {

                            System.out.println("Error reading file");
                            System.out.print("Exception:" + e + "\n");
                            LOGGER.log(Level.SEVERE, "Exception occur", e);
                            //  String err_msg = "file_syspk-"+ max_file_syspk +":"+ ExceptionUtils.getStackTrace(e);

                            String err_msg = "file_syspk-" + max_file_syspk + ":" + e.toString();

                            //inserting error message
                 /*   String sql_error_msg = " INSERT INTO fw_core.fw_log_app_errorlog (client_id, function_id,  error_msg,    create_timestamp, update_timestamp)\n" +
                                            "VALUES(4, 4, ?, now(),now())";
                    PreparedStatement stmt_err_msg = connection2.prepareStatement(sql_error_msg);
                    stmt_err_msg.setString(1,err_msg);
                    stmt_err_msg.executeUpdate();*/
                            //NA updates
                            CallableStatement errorlogFunc = connection2.prepareCall("{ call fw_insert_app_error(?,?,null,?,null,null,null) }");
                            errorlogFunc.setInt(1, client_id);
                            errorlogFunc.setInt(2, function_id);
                            errorlogFunc.setString(3, err_msg);
                            errorlogFunc.execute();
                            errorlogFunc.close();

                   /* String sql_sheet_level2 = "  update fw_core.fw_jobctl_file_sheet_runschedule set end_status='error',end_status_note='stg1_failed' , end_time=now() where file_syspk=?  ";
                    PreparedStatement stmt8 = connection.prepareStatement(sql_sheet_level2);
                    stmt8.setInt(1, max_file_syspk);
                    stmt8.executeUpdate();*/

                            String sql_file_level3 = "  update fw_core.fw_jobctl_file_runschedule set end_status='error',end_status_note='stg1_failed' , end_time=now() where file_syspk=? ";
                            PreparedStatement stmt9 = connection.prepareStatement(sql_file_level3);
                            stmt9.setInt(1, max_file_syspk);
                            stmt9.executeUpdate();

                            String runsch_update = "  update fw_core.fw_jobctl_runschedule set end_status='error',end_status_note='stg1_failed', end_time=now() where latest_runschedule_flag='1'; ";
                            Statement stmt10 = connection.createStatement();
                            stmt10.executeUpdate(runsch_update);

                            //update end_status in fw_jobctl_runschedule_jobstep
                            String sql_file_level4 = "  update fw_core.fw_jobctl_runschedule_jobstep set end_status='error',end_status_note='stg1_failed', end_time=now() where latest_runschedule_flag='1' and run_schedule_id=? and job_id=? and step_id=?;";
                            PreparedStatement stmt11 = connection.prepareStatement(sql_file_level4);
                            stmt11.setInt(1, max_run_schedule_id);
                            stmt11.setInt(2, job_id);
                            stmt11.setInt(3, step_id);
                            stmt11.executeUpdate();


                            continue;
                        }

                        //update sheet_mnemonic in fw_jobctl_file_sheet_runschedule
                        String sheet_mnemonic_update = " update fw_core.fw_jobctl_file_sheet_runschedule a set sheet_format_mnemonic = b.sheet_format,file_sheet_mnemonic=b.file_sheet_mnemonic\n" +
                                "from fw_core.test_file_sheet_format_master b where a.file_syspk =? and a.file_mnemonic=b.file_mnemonic and trim(lower(a.sheet_name)) = lower(b.sheet_name) ;";
                        PreparedStatement stmt5 = connection.prepareStatement(sheet_mnemonic_update);
                        stmt5.setInt(1, max_file_syspk);
                        stmt5.executeUpdate();
                        //  System.out.println("SQL"+sheet_mnemonic_update+"stmt5:"+stmt5);

                        String sheet_mnemonic_update2 = " update fw_core.fw_jobctl_file_sheet_runschedule a set sheet_format_mnemonic = b.sheet_format,file_sheet_mnemonic=b.file_sheet_mnemonic\n" +
                                "from fw_core.test_file_sheet_format_master b where a.file_syspk =? and a.file_mnemonic=b.file_mnemonic and split_part(lower(a.sheet_name),'-',1) = lower(b.sheet_name) and a.file_sheet_mnemonic is null;";
                        PreparedStatement stmt6 = connection.prepareStatement(sheet_mnemonic_update2);
                        stmt6.setInt(1, max_file_syspk);
                        stmt6.executeUpdate();

                        String sheet_mnemonic_update3 = " update fw_core.fw_jobctl_file_sheet_runschedule a set sheet_format_mnemonic = b.sheet_format,file_sheet_mnemonic=b.file_sheet_mnemonic\n" +
                                "from fw_core.test_file_sheet_format_master b where a.file_syspk =? and a.file_mnemonic='BUDNI' and split_part(lower(a.sheet_name),' ',1) = split_part(lower(b.sheet_name),' ',1) ;";
                        PreparedStatement stmt7 = connection.prepareStatement(sheet_mnemonic_update3);
                        stmt7.setInt(1, max_file_syspk);
                        stmt7.executeUpdate();
                    }
                    //update file count and end_status in fw_jobctl_runschedule
                    String sql_file_level3 = "  update fw_core.fw_jobctl_runschedule set end_status='success',end_status_note='stg1_completed', object_count=? , end_time=now() where latest_runschedule_flag='1' and run_schedule_id=? ;";
                    PreparedStatement stmt9 = connection.prepareStatement(sql_file_level3);
                    stmt9.setInt(1, file_count);
                    stmt9.setInt(2, max_run_schedule_id);
                    stmt9.executeUpdate();

                    //update end_status if file load is success in fw_jobctl_runschedule_jobstep
                    String sql_file_level4 = "  update fw_core.fw_jobctl_runschedule_jobstep set end_status='success',end_status_note='stg1_completed', end_time=now() where latest_runschedule_flag='1' and run_schedule_id=? and job_id=? and step_id=?;";
                    PreparedStatement stmt11 = connection.prepareStatement(sql_file_level4);
                    stmt11.setInt(1, max_run_schedule_id);
                    stmt11.setInt(2, job_id);
                    stmt11.setInt(3, step_id);
                    stmt11.executeUpdate();

                    connection.commit();


                    connection.close();
                    connection2.close();

                } catch (SQLException ex2) {
                    System.out.println("Database error");
                    ex2.printStackTrace();
                } catch (NullPointerException e) {
                    e.printStackTrace();
                }

            } else {
                try {
                    con = DriverManager.getConnection(jdbcURL, username, password);
                    con.setAutoCommit(false);
                } catch (SQLException throwables) {
                    throwables.printStackTrace();
                }
                System.out.println("There is no Excel file to process");
                String sql_file_level3 = "  update fw_core.fw_jobctl_runschedule set end_status='success',end_status_note='no files in landing', end_time=now() where latest_runschedule_flag='1' ";
                Statement st = null;
                try {
                    st = con.createStatement();
                    st.executeUpdate(sql_file_level3);
                } catch (SQLException throwables) {
                    throwables.printStackTrace();
                }
                String sql_file_level4 = "  update fw_core.fw_jobctl_runschedule_jobstep set end_status='success',end_status_note='no files in landing', end_time=now() where latest_runschedule_flag='1' and job_script_type='java'";
                Statement st1 = null;
                try {
                    st1 = con.createStatement();
                    st1.executeUpdate(sql_file_level4);
                } catch (SQLException throwables) {
                    throwables.printStackTrace();
                }
                con.commit();
                con.close();
            }
        }

    }

    public static class ExcelFileFilter implements java.io.FileFilter {
        public boolean accept(File file) {
            return file != null &&
                    file.isFile() &&
                    file.canRead() &&
                    (file.getName().endsWith("xls")
                            || file.getName().endsWith("xlsx"));
        }
    }

    private static boolean isRowEmpty(Row row) {
        boolean isEmpty = true;
        DataFormatter dataFormatter = new DataFormatter();

        if (row != null) {
            for (Cell cell : row) {
                if (dataFormatter.formatCellValue(cell).trim().length() > 0) {
                    isEmpty = false;
                    break;
                }
            }
        }

        return isEmpty;
    }

/*String getMergedRegionStringValue(Sheet sheet, int firstRow, int firstColumn){
        for(int i = 0; i < sheet.getNumMergedRegions(); i++) {
            CellRangeAddress region = sheet.getMergedRegion(i);

            int colIndex = region.getFirstColumn();
            int rowNum = region.getFirstRow();
            //check first cell of the region
            if(rowNum == firstRow && colIndex == firstColumn){
                return sheet.getRow(rowNum).getCell(colIndex).getStringCellValue();
            }
        }
    }*/
}

