deason 1 жил өмнө
parent
commit
e596fdf527

+ 3 - 0
src/main/java/cn/com/qmth/examcloud/tool/service/batch_create_user/BatchCreateUserTask.java

@@ -63,6 +63,9 @@ public class BatchCreateUserTask implements TaskService {
 
 
         int total = dataList.size();
         int total = dataList.size();
         log.info("共{}条数据!", total);
         log.info("共{}条数据!", total);
+        if (total == 0) {
+            return;
+        }
 
 
         // 检验数据
         // 检验数据
         Map<String, Long> orgMaps = new HashMap<>();
         Map<String, Long> orgMaps = new HashMap<>();

+ 66 - 11
src/main/java/cn/com/qmth/examcloud/tool/service/batch_import_exam_student/BatchImportExamStudentTask.java

@@ -23,12 +23,18 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashMap;
 import java.util.List;
 import java.util.List;
 import java.util.Map;
 import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
 
 
 @Component
 @Component
 public class BatchImportExamStudentTask implements TaskService {
 public class BatchImportExamStudentTask implements TaskService {
 
 
     private final static Logger log = LoggerFactory.getLogger(BatchImportExamStudentTask.class);
     private final static Logger log = LoggerFactory.getLogger(BatchImportExamStudentTask.class);
 
 
+    private ExecutorService threadService = Executors.newCachedThreadPool();
+
     @Autowired
     @Autowired
     private SysProperty sysProperty;
     private SysProperty sysProperty;
 
 
@@ -63,7 +69,9 @@ public class BatchImportExamStudentTask implements TaskService {
 
 
         int total = dataList.size();
         int total = dataList.size();
         log.info("共{}条数据!examId:{}", total, examId);
         log.info("共{}条数据!examId:{}", total, examId);
-        long start = System.currentTimeMillis();
+        if (total == 0) {
+            return;
+        }
 
 
         // 检验数据
         // 检验数据
         Map<String, Long> orgMaps = new HashMap<>();
         Map<String, Long> orgMaps = new HashMap<>();
@@ -129,25 +137,39 @@ public class BatchImportExamStudentTask implements TaskService {
         if (!errMessages.isEmpty()) {
         if (!errMessages.isEmpty()) {
             throw new StatusException(StringUtils.join(errMessages, "\n"));
             throw new StatusException(StringUtils.join(errMessages, "\n"));
         }
         }
+        long start = System.currentTimeMillis();
 
 
-        int failCount = 0;
+        int batchNum = 20;
+        int runCount = 0;
+        AtomicInteger failCount = new AtomicInteger();
+        List<ExamStudentInfo> batchList = new ArrayList<>();
         for (int i = 0; i < total; i++) {
         for (int i = 0; i < total; i++) {
-            ExamStudentInfo examStudent = dataList.get(i);
+            batchList.add(dataList.get(i));
+            runCount++;
 
 
-            try {
-                this.saveStudent(loginUser, examStudent);
-                log.info("共{}条 第{}条 已执行!{}", total, i + 1, examStudent.getStudentCode());
-            } catch (Exception e) {
-                failCount++;
-                log.error("共{}条 第{}条 执行失败!{} err:{}", total, i + 1, examStudent.getStudentCode(), e.getMessage());
+            if (i % batchNum == 0) {
+                this.concurrentRun(loginUser, batchList, failCount);
+                batchList.clear();
+
+                long cost = (System.currentTimeMillis() - start) / 1000L;
+                log.info("总数:{} 成功数:{} 失败数:{} 平均{}条每秒 已耗时:{}秒 当前第{}条 进度:{}%"
+                        , total, total - failCount.get(), failCount.get(), runCount / (cost + 1)
+                        , cost, runCount, runCount * 100f / total);
             }
             }
         }
         }
 
 
+        if (!batchList.isEmpty()) {
+            this.concurrentRun(loginUser, batchList, failCount);
+        }
+
         long cost = (System.currentTimeMillis() - start) / 1000;
         long cost = (System.currentTimeMillis() - start) / 1000;
-        long rate = total / cost;
-        String msg = String.format("共%s条 成功%s条 失败%s条 耗时%s秒 平均%s条/秒", total, total - failCount, failCount, cost, rate);
+        String msg = String.format("总数:%s 成功数:%s 失败数:%s 平均%s条每秒 总耗时:%s秒"
+                , total, total - failCount.get(), failCount.get(), total / (cost + 1), cost);
         log.info(msg);
         log.info(msg);
         task.setDescription(msg);
         task.setDescription(msg);
+
+        batchList.clear();
+        dataList.clear();
     }
     }
 
 
     private void saveStudent(User loginUser, ExamStudentInfo examStudent) {
     private void saveStudent(User loginUser, ExamStudentInfo examStudent) {
@@ -176,4 +198,37 @@ public class BatchImportExamStudentTask implements TaskService {
         }
         }
     }
     }
 
 
+    private void concurrentRun(User loginUser, List<ExamStudentInfo> batchList, AtomicInteger failCount) {
+        if (batchList.isEmpty()) {
+            return;
+        }
+
+        final CountDownLatch commander = new CountDownLatch(1);
+        final CountDownLatch worker = new CountDownLatch(batchList.size());
+
+        for (int n = 0; n < batchList.size(); n++) {
+            ExamStudentInfo examStudent = batchList.get(n);
+
+            Runnable runnable = () -> {
+                try {
+                    this.saveStudent(loginUser, examStudent);
+                } catch (Exception e) {
+                    failCount.incrementAndGet();
+                    log.error("学号:{} 执行失败!{}", examStudent.getStudentCode(), e.getMessage());
+                } finally {
+                    worker.countDown();
+                }
+            };
+
+            threadService.execute(runnable);
+        }
+
+        try {
+            commander.countDown();
+            worker.await();
+        } catch (Exception e) {
+            log.error(e.getMessage());
+        }
+    }
+
 }
 }

+ 3 - 3
src/test/java/cn/com/qmth/examcloud/tool/DataTest.java

@@ -16,12 +16,12 @@ public class DataTest {
 
 
         List<List<String>> excelRows = new ArrayList<>();
         List<List<String>> excelRows = new ArrayList<>();
         for (int i = 1; i <= 100; i++) {
         for (int i = 1; i <= 100; i++) {
-            long index = 1000000000L + i;
+            String code = "S" + (100000000L + i);
 
 
             List<String> data = new ArrayList<>();
             List<String> data = new ArrayList<>();
             data.add("考生_" + i);
             data.add("考生_" + i);
-            data.add(String.valueOf(index));
-            data.add(String.valueOf(index));
+            data.add(code);
+            data.add(code);
             data.add("org101");
             data.add("org101");
             data.add("");
             data.add("");
             data.add("c101");
             data.add("c101");