添加切割离线文件历史数据小工具

src/main/java/cn/com/qmth/dp/examcloud/oe/modules/cut_exam_offline_data/CutExamOfflineDataService.java

@@ -0,0 +1,136 @@
+package cn.com.qmth.dp.examcloud.oe.modules.cut_exam_offline_data;
+
+import cn.com.qmth.examcloud.commons.logging.ExamCloudLog;
+import cn.com.qmth.examcloud.commons.logging.ExamCloudLogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.stereotype.Service;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/*
+ * @Description 切割离线文件的旧数据，上线时，一次性使用，不能重复执行
+ * @Author lideyin
+ * @Date 2020/9/11 10:51
+ * @Version 1.0
+ */
+@Service
+public class CutExamOfflineDataService {
+    private static ExamCloudLog log = ExamCloudLogFactory.getLog(CutExamOfflineDataService.class);
+
+    @Autowired
+    JdbcTemplate jdbcTemplate;
+    BlockingQueue workQueue = new ArrayBlockingQueue(1000);
+
+    @Async
+    public void start() {
+        String sql = "select exam_record_data_id,offline_file_name,offline_file_url,update_time " +
+                "from ec_oe_exam_record_4_marking where offline_file_name is not null";
+
+        try {
+            Connection connection = jdbcTemplate.getDataSource().getConnection();
+            //流式读取数据
+            PreparedStatement ps = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY,
+                    ResultSet.CONCUR_READ_ONLY);
+            ps.setFetchSize(Integer.MIN_VALUE);
+            ResultSet rs = ps.executeQuery(sql);
+
+            int count = 0;
+            final AtomicInteger batchCount = new AtomicInteger(0);
+            //待插入的数据
+            List<Object[]> dataList = new ArrayList<>();
+
+            ThreadPoolExecutor executor =
+                    new ThreadPoolExecutor(4, 4, 30, TimeUnit.SECONDS, workQueue);
+            while (rs.next()) {
+                Long examRecordDataId = rs.getLong("exam_record_data_id");
+                String offlineFileName = rs.getString("offline_file_name");
+                String offlineFileUrl = rs.getString("offline_file_url");
+                String updateTime = rs.getString("update_time");
+                String creationTime = updateTime;
+                String originalFileName = offlineFileName;
+
+                String fileType = "";
+                //格式不正确的数据直接跳过
+                if (offlineFileName.indexOf(".") == -1) {
+                    String errMsg = String.format("examRecordDataId=%s的数据有误，无扩展名.offlineFileName=%s",
+                            examRecordDataId, offlineFileName);
+                    System.out.println(errMsg);
+                    log.error(errMsg);
+                    continue;
+                }
+                fileType = offlineFileName.substring(offlineFileName.indexOf(".") + 1);
+                String suffix = fileType;
+
+                dataList.add(new Object[]
+                        {examRecordDataId, offlineFileName, originalFileName,
+                                offlineFileUrl, fileType, suffix, creationTime, updateTime});
+                count++;
+
+                log.info("[CutExamOfflineDataService]. count=" + count);
+
+                //每500条数据执行一次批量插入操作
+                if (count % 500 == 0) {
+                    List<Object[]> finalDataList = new ArrayList<>();
+                    finalDataList.addAll(dataList);
+                    executor.execute(new Runnable() {
+                        @Override
+                        public void run() {
+                            batchAddOfflineData(finalDataList);
+                            batchCount.incrementAndGet();
+                            log.info(String.format("[CutExamOfflineDataService] 执行第%s 批次数据操作", batchCount));
+                        }
+                    });
+
+                    count = 0;
+                    dataList = new ArrayList<>();
+                }
+            }
+            log.info("数据遍历完成，多线程插入数据中...");
+
+            //剩下的最后一批次的数据单独执行
+            if (dataList.size() > 0) {
+
+                List<Object[]> finalDataList = new ArrayList<>();
+                finalDataList.addAll(dataList);
+                executor.execute(new Runnable() {
+                    @Override
+                    public void run() {
+                        batchAddOfflineData(finalDataList);
+                        batchCount.incrementAndGet();
+                        log.info(String.format("all is over?[CutExamOfflineDataService] 执行第%s 批次数据操作", batchCount));
+                    }
+                });
+
+            }
+            executor.shutdown();
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * 批量添加离线文件数据
+     *
+     * @param batchArgs
+     */
+    private void batchAddOfflineData(List<Object[]> batchArgs) {
+        String strSql = "INSERT INTO ec_oe_exam_record_file_answer " +
+                "(exam_record_data_id,file_name,original_file_name,file_url,file_type,suffix,creation_time,update_time) " +
+                "VALUES (?,?,?,?,?,?,?,?)";
+        jdbcTemplate.batchUpdate(strSql, batchArgs);
+    }
+
+
+}