优化切割离线文件历史数据小工具日志

src/main/java/cn/com/qmth/dp/examcloud/oe/modules/cut_exam_offline_data/CutExamOfflineDataService.java

@@ -2,6 +2,7 @@ package cn.com.qmth.dp.examcloud.oe.modules.cut_exam_offline_data;
 
 import cn.com.qmth.examcloud.commons.logging.ExamCloudLog;
 import cn.com.qmth.examcloud.commons.logging.ExamCloudLogFactory;
+import org.apache.poi.ss.formula.functions.Count;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.scheduling.annotation.Async;
@@ -13,10 +14,7 @@ import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicInteger;
 
 /*
@@ -32,6 +30,9 @@ public class CutExamOfflineDataService {
     @Autowired
     JdbcTemplate jdbcTemplate;
     BlockingQueue workQueue = new ArrayBlockingQueue(1000);
+    //批次数量
+    static Integer BATCH_NUMBER = 500;
+    CountDownLatch countDownLatch;
 
     @Async
     public void start() {
@@ -39,13 +40,30 @@ public class CutExamOfflineDataService {
                 "from ec_oe_exam_record_4_marking where offline_file_name is not null";
 
         try {
+            //校验本程序是否为第一次执行，是则执行，否则抛错
+            Integer targetCount = jdbcTemplate.queryForObject(
+                    "select count(*) from ec_oe_exam_record_file_answer", Integer.class);
+            if (targetCount.intValue() > 0) {
+                log.error(String.format("[CutExamOfflineDataService] 本程序不允许重复执行！若要重复执行，请确保ec_oe_exam_record_file_answer表中无数据，当前共有%s条数据",targetCount));
+                return;
+            }
+
+            //总行数
+            Integer totalCount = jdbcTemplate.queryForObject(
+                    "select count(*) from ec_oe_exam_record_4_marking where offline_file_name is not null", Integer.class);
+            //总执行批次数
+            Integer totalBatchCount = totalCount % BATCH_NUMBER == 0
+                    ? totalCount / BATCH_NUMBER
+                    : (totalCount / BATCH_NUMBER + 1);
+            //初始化countDownLatch
+            countDownLatch = new CountDownLatch(totalBatchCount);
 
             Connection connection = jdbcTemplate.getDataSource().getConnection();
             //流式读取数据
             PreparedStatement ps = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY,
                     ResultSet.CONCUR_READ_ONLY);
             ps.setFetchSize(Integer.MIN_VALUE);
-            log.info("开始流式读取数据....");
+            log.info("[CutExamOfflineDataService] 开始流式读取数据....");
             ResultSet rs = ps.executeQuery(sql);
 
             int count = 0;
@@ -66,7 +84,7 @@ public class CutExamOfflineDataService {
                 String fileType = "";
                 //格式不正确的数据直接跳过
                 if (offlineFileName.indexOf(".") == -1) {
-                    String errMsg = String.format("examRecordDataId=%s的数据有误，无扩展名.offlineFileName=%s",
+                    String errMsg = String.format("[CutExamOfflineDataService] examRecordDataId=%s的数据有误，无扩展名.offlineFileName=%s",
                             examRecordDataId, offlineFileName);
                     System.out.println(errMsg);
                     log.error(errMsg);
@@ -82,8 +100,9 @@ public class CutExamOfflineDataService {
 
 //                log.info("[CutExamOfflineDataService]. count=" + count);
 
+
                 //每500条数据执行一次批量插入操作
-                if (count % 500 == 0) {
+                if (count % BATCH_NUMBER == 0) {
                     List<Object[]> finalDataList = new ArrayList<>();
                     finalDataList.addAll(dataList);
                     executor.execute(new Runnable() {
@@ -91,14 +110,15 @@ public class CutExamOfflineDataService {
                         public void run() {
                             batchAddOfflineData(finalDataList);
                             batchCount.incrementAndGet();
-                            log.info(String.format("[CutExamOfflineDataService] 执行第%s 批次数据操作", batchCount));
+                            log.info(String.format("[CutExamOfflineDataService] 共%s个批次,执行第%s 批次数据操作",totalBatchCount, batchCount));
+                            countDownLatch.countDown();
                         }
                     });
 
                     dataList = new ArrayList<>();
                 }
             }
-            log.info(String.format("数据遍历完成，多线程插入数据中...共计%s条数据，共需执行%s个批次的插入动作", count, (count / 500 + 1)));
+            log.info(String.format("[CutExamOfflineDataService] 数据遍历完成，多线程插入数据中...共计%s条数据，共需执行%s个批次的插入动作", count, totalBatchCount));
 
             //剩下的最后一批次的数据单独执行
             if (dataList.size() > 0) {
@@ -110,13 +130,21 @@ public class CutExamOfflineDataService {
                     public void run() {
                         batchAddOfflineData(finalDataList);
                         batchCount.incrementAndGet();
-                        log.info(String.format("all is over?[CutExamOfflineDataService] 执行第%s 批次数据操作", batchCount));
+                        log.info(String.format("[CutExamOfflineDataService]  共%s个批次,执行第%s 批次数据操作",totalBatchCount, batchCount));
+                        countDownLatch.countDown();
                     }
                 });
 
             }
             executor.shutdown();
+            log.info("[CutExamOfflineDataService] waiting child thread...........");
+            countDownLatch.await();
+            log.info("[CutExamOfflineDataService] all is over...........");
         } catch (SQLException e) {
+            log.error("[CutExamOfflineDataService] 执行过程中出现异常",e);
+            throw new RuntimeException(e);
+        } catch (InterruptedException e) {
+            log.error("[CutExamOfflineDataService] 执行过程中出现异常",e);
             throw new RuntimeException(e);
         }
     }