diff --git a/src/main/java/com/batch/config/AsyncConfig.java b/src/main/java/com/batch/config/AsyncConfig.java index d52a3f2..a97c849 100644 --- a/src/main/java/com/batch/config/AsyncConfig.java +++ b/src/main/java/com/batch/config/AsyncConfig.java @@ -2,6 +2,7 @@ package com.batch.config; import java.util.concurrent.Executor; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.scheduling.annotation.EnableAsync; @@ -10,13 +11,22 @@ import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; @Configuration @EnableAsync public class AsyncConfig { + + @Value("${thread.comm.count}") + Integer iCommAsyncThreadCount; + @Value("${thread.ext.count}") + Integer iExtAsyncThreadCount; + @Value("${thread.ai.count}") + Integer iAiAsyncThreadCount; + @Bean(name = "commAsync") public Executor commAsyncExecutor() { ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); - executor.setCorePoolSize(5); - executor.setMaxPoolSize(50); - executor.setKeepAliveSeconds(30); + executor.setCorePoolSize(iCommAsyncThreadCount); + executor.setMaxPoolSize(iCommAsyncThreadCount); + executor.setQueueCapacity(1000); + executor.setKeepAliveSeconds(15); executor.setAllowCoreThreadTimeOut(true); executor.setWaitForTasksToCompleteOnShutdown(true); executor.setThreadNamePrefix("commAsync-processor-"); @@ -25,13 +35,28 @@ public class AsyncConfig { return executor; } + @Bean(name = "extAsync") + public Executor extAsyncExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(iExtAsyncThreadCount); + executor.setMaxPoolSize(iExtAsyncThreadCount); + executor.setQueueCapacity(1000); + executor.setKeepAliveSeconds(15); + executor.setAllowCoreThreadTimeOut(true); + executor.setWaitForTasksToCompleteOnShutdown(true); + executor.setThreadNamePrefix("extAsync-processor-"); + executor.initialize(); + + return executor; + } + @Bean(name = "aiAsync") public Executor aiAsyncExecutor() { ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); - executor.setCorePoolSize(50); - executor.setMaxPoolSize(100); + executor.setCorePoolSize(iAiAsyncThreadCount); + executor.setMaxPoolSize(iAiAsyncThreadCount); executor.setQueueCapacity(1000); - executor.setKeepAliveSeconds(30); + executor.setKeepAliveSeconds(15); executor.setAllowCoreThreadTimeOut(true); executor.setWaitForTasksToCompleteOnShutdown(true); executor.setThreadNamePrefix("aiAsync-processor-"); diff --git a/src/main/java/com/batch/config/MatchingExtraProcessorAuto.java b/src/main/java/com/batch/config/MatchingExtraProcessorAuto.java index 44548dd..98bc6e6 100644 --- a/src/main/java/com/batch/config/MatchingExtraProcessorAuto.java +++ b/src/main/java/com/batch/config/MatchingExtraProcessorAuto.java @@ -42,11 +42,13 @@ public class MatchingExtraProcessorAuto { String sAccnutYm = (String) paramRec.get("accnut_ym"); String sCprCode = (String) paramRec.get("cpr_code"); String sPartCpr = (String) paramRec.get("partn_cpr"); + String sConds = (String) paramRec.get("conds"); //작업시작 Map mParam = new HashMap(); mParam.put("sysSe", sSysSe); mParam.put("accnutYm", sAccnutYm); + mParam.put("conds", sConds); //---------------------------------------------------------------------------- //자기법인 데이타 가져오기 diff --git a/src/main/java/com/batch/controller/JobController.java b/src/main/java/com/batch/controller/JobController.java index 99339b0..38cbf82 100644 --- a/src/main/java/com/batch/controller/JobController.java +++ b/src/main/java/com/batch/controller/JobController.java @@ -4,9 +4,11 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; @@ -20,6 +22,7 @@ import com.batch.service.JobService; import com.batch.util.FileUtil; import com.google.gson.JsonObject; +import com.batch.controller.JobController; import lombok.extern.slf4j.Slf4j; @RestController @@ -34,49 +37,69 @@ public class JobController { @PostMapping("/create") - public String createJob( @RequestBody Map params) throws Exception { + public Map createJob( @RequestBody Map params) throws Exception { /* * { * "sysSe": "LS_ALL", * "accnutYm": "202306", * } - */ + */ + //Job Create Log + UUID uuid = UUID.randomUUID(); + String sJobGroup = uuid.toString(); + log.debug("Start Create Job"); - jobService.createData(params); + jobService.createData(sJobGroup, params); log.debug("End Create Job"); + Map rtnMap = new HashMap(); + rtnMap.put("jobGroupId", sJobGroup); + rtnMap.put("jobMessage", "신규 작업데이타를 생성합니다. 작업이 끝난후 작업결과는 별도로 확인 바랍니다."); - return "신규 작업데이타를 생성합니다. 작업이 끝난후 작업결과는 별도로 확인 바랍니다."; + return rtnMap; } @PostMapping("/matching") - public String matchingJob( @RequestBody Map params) throws Exception { + public Map matchingJob( @RequestBody Map params) throws Exception { /* * { * "sysSe": "LS_ALL", * "accnutYm": "202306", * } - */ + */ + //Job Create Log + UUID uuid = UUID.randomUUID(); + String sJobGroup = uuid.toString(); + log.debug("Start Matching Job"); - jobService.matchingJob(params); + jobService.matchingJob(sJobGroup, params); log.debug("End Matching Job"); - - return "매칭작업을 시작합니다. 작업이 끝난후 작업결과는 별도로 확인 바랍니다."; + Map rtnMap = new HashMap(); + rtnMap.put("jobGroupId", sJobGroup); + rtnMap.put("jobMessage", "매칭작업을 시작합니다. 작업이 끝난후 작업결과는 별도로 확인 바랍니다."); + + return rtnMap; } @PostMapping("/extramatching") - public String extraMatchingJob( @RequestBody Map params) throws Exception { + public Map extraMatchingJob( @RequestBody Map params) throws Exception { /* * { * "sysSe": "LS_ALL", - * "accnutYm": "202306", + * "accnutYm": "202306", + * "searchOne": "dta_ty in ('11','21','31','33','35','37','41')", + * "searchTwo": "dta_ty in ('11','21','31','33','35','37','41')" * } - */ + */ + //Job Create Log + UUID uuid = UUID.randomUUID(); + String sJobGroup = uuid.toString(); + log.debug("Start Extra Matching Job"); List retData = matchingInnerDelingMapper.getCustomItemReadData(params); // List retData = new ArrayList(); @@ -84,53 +107,79 @@ public class JobController { // m.put("sys_se", "LS_ALL"); // m.put("accnut_ym", "202311"); // m.put("cpr_code", "A15300"); -// m.put("partn_cpr", "A01100"); +// m.put("partn_cpr", "A01100"); +// m.put("searchOne", "dta_ty in ('11','21','41')"); +// m.put("searchTwo", "dta_ty in ('12','22','42')"); // retData.add(m); + // 수익/비용 + for(Map curMap : retData) { + curMap.put("conds", "T"); + jobService.extraJobSub(sJobGroup, curMap); + } + //채권/채무 for(Map curMap : retData) { - jobService.extraJobSub(curMap); + curMap.put("conds", "B"); + jobService.extraJobSub(sJobGroup, curMap); } + log.debug("End Extra Matching Job"); + Map rtnMap = new HashMap(); + rtnMap.put("jobGroupId", sJobGroup); + rtnMap.put("jobMessage", "Extra 매칭작업을 시작합니다. 작업이 끝난후 작업결과는 별도로 확인 바랍니다."); - return "Extra 매칭작업을 시작합니다. 작업이 끝난후 작업결과는 별도로 확인 바랍니다."; + return rtnMap; } @PostMapping("/aimatching") - public String aiMatchingJob( @RequestBody Map params) throws Exception { + public Map aiMatchingJob( @RequestBody Map params) throws Exception { /* * { * "sysSe": "LS_ALL", * "accnutYm": "202306", * } - */ + */ + //Job Create Log + UUID uuid = UUID.randomUUID(); + String sJobGroup = uuid.toString(); + log.debug("Start AI Matching Job"); List retData = matchingInnerDelingMapper.getAiReadData(params); for(Map curMap : retData) { - jobService.aiJobSub(curMap); + jobService.aiJobSub(sJobGroup, curMap); } log.debug("End AI Matching Job"); + Map rtnMap = new HashMap(); + rtnMap.put("jobGroupId", sJobGroup); + rtnMap.put("jobMessage", "AI 매칭작업을 시작합니다. 작업이 끝난후 작업결과는 별도로 확인 바랍니다."); - return "AI 매칭작업을 시작합니다. 작업이 끝난후 작업결과는 별도로 확인 바랍니다."; + return rtnMap; } @PostMapping("/return") - public String returnJob( @RequestBody Map params) throws Exception { + public Map returnJob( @RequestBody Map params) throws Exception { /* * { * "sysSe": "LS_ALL", * "accnutYm": "202306", * } - */ + */ + UUID uuid = UUID.randomUUID(); + String sJobGroup = uuid.toString(); + log.debug("Start Return Job"); - jobService.returnRwsultData(params); + jobService.returnRwsultData(sJobGroup, params); log.debug("End Return Job"); + Map rtnMap = new HashMap(); + rtnMap.put("jobGroupId", sJobGroup); + rtnMap.put("jobMessage", "매칭결과 반영을 시작합니다. 작업이 끝난후 작업결과는 별도로 확인 바랍니다."); - return "매칭결과 반영을 시작합니다. 작업이 끝난후 작업결과는 별도로 확인 바랍니다."; + return rtnMap; } @@ -179,5 +228,27 @@ public class JobController { return lJobList; } + @GetMapping("/userJobInfo/{userJobGroup}") + public Object statusBatchJob(@PathVariable(name = "userJobGroup") String sUserJobGroup) throws Exception { + Map rtnVal = new HashMap(); + + Map mParam = new HashMap(); + mParam.put("userJobGroup", sUserJobGroup); + List lmJob = matchingInnerDelingMapper.getUserJobStatus(mParam); + rtnVal.put("JobList", lmJob); + boolean blnStatus = true; + for (Map curMap : lmJob) { + String ScurStatus = (String) curMap.get("status"); + if (!"Finished".equalsIgnoreCase(ScurStatus)) { + blnStatus = false; + break; + } + } + rtnVal.put("JobStatus", blnStatus); + + + return rtnVal; + } + } diff --git a/src/main/java/com/batch/mapper/primary/MatchingInnerDelingMapper.java b/src/main/java/com/batch/mapper/primary/MatchingInnerDelingMapper.java index 5c4eb33..d6034aa 100644 --- a/src/main/java/com/batch/mapper/primary/MatchingInnerDelingMapper.java +++ b/src/main/java/com/batch/mapper/primary/MatchingInnerDelingMapper.java @@ -100,6 +100,14 @@ public interface MatchingInnerDelingMapper { */ @SuppressWarnings("rawtypes") int insertDataAiFromOriginal(Map param); + + /** + * 새로운 일치키 생성전 초기화 + * @param param + * @return + */ + @SuppressWarnings("rawtypes") + int updateClearNewMatchKey(Map param); /** * 값을 돌려주기전 월별 새로운 일치키 생성 @@ -154,5 +162,11 @@ public interface MatchingInnerDelingMapper { */ @SuppressWarnings("rawtypes") List getAiReadData(Map param); + + /* + * User Job Status + */ + @SuppressWarnings("rawtypes") + List getUserJobStatus(Map param); } \ No newline at end of file diff --git a/src/main/java/com/batch/service/JobService.java b/src/main/java/com/batch/service/JobService.java index a1dbdd9..a409137 100644 --- a/src/main/java/com/batch/service/JobService.java +++ b/src/main/java/com/batch/service/JobService.java @@ -34,6 +34,7 @@ import com.batch.mapper.primary.MatchingInnerDelingMapper; import com.batch.mapper.secondary.OracleMapper; import com.batch.util.FileUtil; +import com.batch.service.JobService; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -61,11 +62,24 @@ public class JobService { @SuppressWarnings("rawtypes") @Async("commAsync") - public void matchingJob( Map params) throws Exception { + public void matchingJob(String jobGroupId, Map params) throws Exception { + //Job Create Log + UUID uuid = UUID.randomUUID(); + HashMap mt = new HashMap(); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-hh-mm-ss"); + String sDate = dateFormat.format(new Date()) + ":" + uuid.toString(); + + Map paramLog = new HashMap(); + paramLog.put("user_job_group", jobGroupId); + paramLog.put("user_job_id", sDate); + paramLog.put("user_job_name", "자동매칭(" + params.toString() + ")"); + matchingInnerDelingMapper.createUserJob(paramLog); + + String sThreadName = Thread.currentThread().getName(); long startTime = System.currentTimeMillis(); - log.info("Job Started : " + startTime); - log.debug("params=" + params.toString()); + log.info("[" + sThreadName + "]Job Started : " + startTime); + log.debug("[" + sThreadName + "]params=" + params.toString()); StringBuffer sb = FileUtil.readFileToString("matchingSetup.json"); MatchingSetup matchingSetup = (MatchingSetup) FileUtil.strToObj(sb.toString(), MatchingSetup.class); @@ -80,14 +94,14 @@ public class JobService { if (matching.getActive()) { lJobType.add(matching.getType()); } else { - log.info("JobType(" + matching.getType() + ") is Disabled"); + log.info("[" + sThreadName + "]JobType(" + matching.getType() + ") is Disabled"); } } } for (String sJobType : lJobType) { - log.info("Current running job type: " + sJobType); + log.info("[" + sThreadName + "]Current running job type: " + sJobType); JobExecution jobExe = invokeJob("matchingInnerDelng", sJobType, params); if (!jobExe.getStatus().equals(BatchStatus.COMPLETED)) { throw new Exception("Job execution error : Batchstatus(" + jobExe.getStatus() + "), ExitStatus(" + jobExe.getExitStatus() + ")" ); @@ -95,20 +109,38 @@ public class JobService { } long endTime = System.currentTimeMillis(); - log.info("Job Type : " + lJobType.toString()); - log.info("Job Ended: " + endTime); - log.info("Running Time : " + (endTime - startTime) + "ms"); + log.info("[" + sThreadName + "]Job Type : " + lJobType.toString()); + log.info("[" + sThreadName + "]Job Ended: " + endTime); + log.info("[" + sThreadName + "]Running Time : " + (endTime - startTime) + "ms"); + //작업종료에 대한 로그 업데이트 + paramLog.put("exit_code", "0"); + paramLog.put("exit_message", ""); + matchingInnerDelingMapper.finishUserJob(paramLog); + } @SuppressWarnings("rawtypes") - @Async("aiAsync") - public void extraJobSub(Map paramRec) throws Exception { + @Async("extAsync") + public void extraJobSub(String jobGroupId, Map paramRec) throws Exception { + + //Job Create Log + UUID uuid = UUID.randomUUID(); + HashMap mt = new HashMap(); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-hh-mm-ss"); + String sDate = dateFormat.format(new Date()) + ":" + uuid.toString(); + + Map paramLog = new HashMap(); + paramLog.put("user_job_group", jobGroupId); + paramLog.put("user_job_id", sDate); + paramLog.put("user_job_name", "Extra매칭(" + paramRec.toString() + ")"); + matchingInnerDelingMapper.createUserJob(paramLog); + String sThreadName = Thread.currentThread().getName(); long startTime = System.currentTimeMillis(); - log.info("Job Started : " + startTime); - log.debug("params=" + paramRec.toString()); + log.info("[" + sThreadName + "]Job Started : " + startTime); + log.debug("[" + sThreadName + "]params=" + paramRec.toString()); MatchingExtraProcessorAuto matchingExtraProcessorAuto = new MatchingExtraProcessorAuto(matchingInnerDelingMapper); @@ -124,30 +156,27 @@ public class JobService { } //3건씩 매칭일 경우 최대 5000건 까지 - for (int i=0; i<5000;i=i+50) { + for (int i=0; i<2000;i=i+100) { matchingExtraProcessorAuto.process(paramRec, 1, 3, 0, i); } - for (int i=0; i<5000;i=i+50) { + for (int i=0; i<2000;i=i+100) { matchingExtraProcessorAuto.process(paramRec, 3, 1, i, 0); } - - //3건씩 매칭일 경우 최대 2000건 까지 - for (int i=0; i<2000;i=i+25) { - matchingExtraProcessorAuto.process(paramRec, 1, 4, 0, i); - } - for (int i=0; i<2000;i=i+25) { - matchingExtraProcessorAuto.process(paramRec, 4, 1, i, 0); - } long endTime = System.currentTimeMillis(); - log.info("Job Ended: " + endTime); - log.info("Running Time : " + (endTime - startTime) + "ms"); + log.info("[" + sThreadName + "]Job Ended: " + endTime); + log.info("[" + sThreadName + "]Running Time : " + (endTime - startTime) + "ms"); + //작업종료에 대한 로그 업데이트 + paramLog.put("exit_code", "0"); + paramLog.put("exit_message", ""); + matchingInnerDelingMapper.finishUserJob(paramLog); + } @SuppressWarnings("rawtypes") @Async("aiAsync") - public void aiJobSub(Map paramRec) throws Exception { + public void aiJobSub(String jobGroupId, Map paramRec) throws Exception { //Job Create Log @@ -157,6 +186,7 @@ public class JobService { String sDate = dateFormat.format(new Date()) + ":" + uuid.toString(); Map paramLog = new HashMap(); + paramLog.put("user_job_group", jobGroupId); paramLog.put("user_job_id", sDate); paramLog.put("user_job_name", "AI매칭(" + paramRec.toString() + ")"); matchingInnerDelingMapper.createUserJob(paramLog); @@ -171,10 +201,12 @@ public class JobService { String sCprCode = (String) paramRec.get("cpr_code"); String sPartCpr = (String) paramRec.get("partn_cpr"); String sDelngCrncy = (String) paramRec.get("delng_crncy"); - + + String sThreadName = Thread.currentThread().getName(); + log.debug("call python"); new ProcessExecutor() - .command(sPythonPrg, sPythonAiTarget, sSysSe, sAccnutYm, sCprCode, sPartCpr, sDelngCrncy) + .command(sPythonPrg, sPythonAiTarget, sThreadName, sSysSe, sAccnutYm, sCprCode, sPartCpr, sDelngCrncy) .redirectOutput(new LogOutputStream() { @Override protected void processLine(String line) { @@ -194,7 +226,6 @@ public class JobService { matchingInnerDelingMapper.finishUserJob(paramLog); } - public JobExecution invokeJob(String jobName, String jobType, Map params) throws JobInstanceAlreadyCompleteException, JobExecutionAlreadyRunningException, JobParametersInvalidException, JobRestartException { @@ -221,7 +252,7 @@ public class JobService { @SuppressWarnings("rawtypes") @Async("commAsync") - public void createData( Map params) throws Exception { + public void createData(String jobGroupId, Map params) throws Exception { //Job Create Log UUID uuid = UUID.randomUUID(); @@ -230,6 +261,7 @@ public class JobService { String sDate = dateFormat.format(new Date()) + ":" + uuid.toString(); Map paramLog = new HashMap(); + paramLog.put("user_job_group", jobGroupId); paramLog.put("user_job_id", sDate); paramLog.put("user_job_name", "작업데이타생성(" + params.toString() + ")"); matchingInnerDelingMapper.createUserJob(paramLog); @@ -294,14 +326,13 @@ public class JobService { paramLog.put("exit_code", "0"); paramLog.put("exit_message", ""); matchingInnerDelingMapper.finishUserJob(paramLog); - } @SuppressWarnings("rawtypes") @Async("commAsync") - public void returnRwsultData( Map params) throws Exception { + public void returnRwsultData(String jobGroupId, Map params) throws Exception { //Job Create Log UUID uuid = UUID.randomUUID(); @@ -310,6 +341,7 @@ public class JobService { String sDate = dateFormat.format(new Date()) + ":" + uuid.toString(); Map paramLog = new HashMap(); + paramLog.put("user_job_group", jobGroupId); paramLog.put("user_job_id", sDate); paramLog.put("user_job_name", "결과데이타리턴(" + params.toString() + ")"); matchingInnerDelingMapper.createUserJob(paramLog); @@ -318,8 +350,10 @@ public class JobService { log.info("Update Data Started : " + startTime); log.debug("params=" + params.toString()); - //기존데이타 삭제 - int iUpdated = matchingInnerDelingMapper.updateNewMatchKey(params); + //기존데이타 초기화 + int iUpdated = matchingInnerDelingMapper.updateClearNewMatchKey(params); + //새로운 매칭키 생성 + iUpdated = matchingInnerDelingMapper.updateNewMatchKey(params); log.debug("Updated OrgData : " + iUpdated + "건"); long endTime = System.currentTimeMillis(); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index f3a5b5e..373b089 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -48,6 +48,11 @@ logging.level.root=info logging.level.com.batch=info logging.level.p6spy=off +#Thread Count 설정 +thread.comm.count=10 +thread.ext.count=50 +thread.ai.count=50 + #Python 프로퍼티 설정 pytyon.path=D:\\Programs\\devp\\python-3.12.2\\python.exe python.ai.target=D:\\Working\\Python\\Test1.py \ No newline at end of file diff --git a/src/main/resources/mybatis/primaryMapper/MatchingInnerDelingMapper.xml b/src/main/resources/mybatis/primaryMapper/MatchingInnerDelingMapper.xml index afbd2d8..501e970 100644 --- a/src/main/resources/mybatis/primaryMapper/MatchingInnerDelingMapper.xml +++ b/src/main/resources/mybatis/primaryMapper/MatchingInnerDelingMapper.xml @@ -361,6 +361,16 @@ cons_group = #{sysSe} and account_period = #{accnutYm} + + + update batch_tbcr_inner_delng + set + new_mtch_ty = null, + new_mtch_ky = null + where + sys_se = #{sysSe} + AND accnut_ym = #{accnutYm} + merge into batch_tbcr_inner_delng m @@ -410,7 +420,12 @@ and cpr_code = #{cprCode} and partn_cpr = #{partnCpr} and mtch_ky is null - and dta_ty in ('11','21','31','33','35','37','41') + + and dta_ty in ('11','21','41') + + + and dta_ty in ('31','33','35','37') + and delng_amt != 0 ORDER BY delng_de @@ -433,7 +448,12 @@ and cpr_code = #{cprCode} and partn_cpr = #{partnCpr} and mtch_ky is null - and dta_ty in ('12','22','32','34','36','38','42') + + and dta_ty in ('12','22','42') + + + and dta_ty in ('32','34','36','38') + and delng_amt != 0 ORDER BY delng_de @@ -456,11 +476,13 @@ INSERT INTO public.batch_user_job_status ( + user_job_group, user_job_id, user_job_name, start_time, status ) VALUES ( + #{user_job_group}, #{user_job_id}, #{user_job_name}, now(), @@ -538,5 +560,68 @@ , partn_cpr , delng_crncy HAVING sum(cnt) > 1 - + + + + + + + + \ No newline at end of file -- libgit2 0.21.4