|
@@ -23,12 +23,18 @@ import java.util.ArrayList;
|
|
|
import java.util.HashMap;
|
|
|
import java.util.List;
|
|
|
import java.util.Map;
|
|
|
+import java.util.concurrent.CountDownLatch;
|
|
|
+import java.util.concurrent.ExecutorService;
|
|
|
+import java.util.concurrent.Executors;
|
|
|
+import java.util.concurrent.atomic.AtomicInteger;
|
|
|
|
|
|
@Component
|
|
|
public class BatchImportExamStudentTask implements TaskService {
|
|
|
|
|
|
private final static Logger log = LoggerFactory.getLogger(BatchImportExamStudentTask.class);
|
|
|
|
|
|
+ private ExecutorService threadService = Executors.newCachedThreadPool();
|
|
|
+
|
|
|
@Autowired
|
|
|
private SysProperty sysProperty;
|
|
|
|
|
@@ -63,7 +69,9 @@ public class BatchImportExamStudentTask implements TaskService {
|
|
|
|
|
|
int total = dataList.size();
|
|
|
log.info("共{}条数据!examId:{}", total, examId);
|
|
|
- long start = System.currentTimeMillis();
|
|
|
+ if (total == 0) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
|
|
|
// 检验数据
|
|
|
Map<String, Long> orgMaps = new HashMap<>();
|
|
@@ -129,25 +137,39 @@ public class BatchImportExamStudentTask implements TaskService {
|
|
|
if (!errMessages.isEmpty()) {
|
|
|
throw new StatusException(StringUtils.join(errMessages, "\n"));
|
|
|
}
|
|
|
+ long start = System.currentTimeMillis();
|
|
|
|
|
|
- int failCount = 0;
|
|
|
+ int batchNum = 20;
|
|
|
+ int runCount = 0;
|
|
|
+ AtomicInteger failCount = new AtomicInteger();
|
|
|
+ List<ExamStudentInfo> batchList = new ArrayList<>();
|
|
|
for (int i = 0; i < total; i++) {
|
|
|
- ExamStudentInfo examStudent = dataList.get(i);
|
|
|
+ batchList.add(dataList.get(i));
|
|
|
+ runCount++;
|
|
|
|
|
|
- try {
|
|
|
- this.saveStudent(loginUser, examStudent);
|
|
|
- log.info("共{}条 第{}条 已执行!{}", total, i + 1, examStudent.getStudentCode());
|
|
|
- } catch (Exception e) {
|
|
|
- failCount++;
|
|
|
- log.error("共{}条 第{}条 执行失败!{} err:{}", total, i + 1, examStudent.getStudentCode(), e.getMessage());
|
|
|
+ if (i % batchNum == 0) {
|
|
|
+ this.concurrentRun(loginUser, batchList, failCount);
|
|
|
+ batchList.clear();
|
|
|
+
|
|
|
+ long cost = (System.currentTimeMillis() - start) / 1000L;
|
|
|
+ log.info("总数:{} 成功数:{} 失败数:{} 平均{}条每秒 已耗时:{}秒 当前第{}条 进度:{}%"
|
|
|
+ , total, total - failCount.get(), failCount.get(), runCount / (cost + 1)
|
|
|
+ , cost, runCount, runCount * 100f / total);
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ if (!batchList.isEmpty()) {
|
|
|
+ this.concurrentRun(loginUser, batchList, failCount);
|
|
|
+ }
|
|
|
+
|
|
|
long cost = (System.currentTimeMillis() - start) / 1000;
|
|
|
- long rate = total / cost;
|
|
|
- String msg = String.format("共%s条 成功%s条 失败%s条 耗时%s秒 平均%s条/秒", total, total - failCount, failCount, cost, rate);
|
|
|
+ String msg = String.format("总数:%s 成功数:%s 失败数:%s 平均%s条每秒 总耗时:%s秒"
|
|
|
+ , total, total - failCount.get(), failCount.get(), total / (cost + 1), cost);
|
|
|
log.info(msg);
|
|
|
task.setDescription(msg);
|
|
|
+
|
|
|
+ batchList.clear();
|
|
|
+ dataList.clear();
|
|
|
}
|
|
|
|
|
|
private void saveStudent(User loginUser, ExamStudentInfo examStudent) {
|
|
@@ -176,4 +198,37 @@ public class BatchImportExamStudentTask implements TaskService {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ private void concurrentRun(User loginUser, List<ExamStudentInfo> batchList, AtomicInteger failCount) {
|
|
|
+ if (batchList.isEmpty()) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ final CountDownLatch commander = new CountDownLatch(1);
|
|
|
+ final CountDownLatch worker = new CountDownLatch(batchList.size());
|
|
|
+
|
|
|
+ for (int n = 0; n < batchList.size(); n++) {
|
|
|
+ ExamStudentInfo examStudent = batchList.get(n);
|
|
|
+
|
|
|
+ Runnable runnable = () -> {
|
|
|
+ try {
|
|
|
+ this.saveStudent(loginUser, examStudent);
|
|
|
+ } catch (Exception e) {
|
|
|
+ failCount.incrementAndGet();
|
|
|
+ log.error("学号:{} 执行失败!{}", examStudent.getStudentCode(), e.getMessage());
|
|
|
+ } finally {
|
|
|
+ worker.countDown();
|
|
|
+ }
|
|
|
+ };
|
|
|
+
|
|
|
+ threadService.execute(runnable);
|
|
|
+ }
|
|
|
+
|
|
|
+ try {
|
|
|
+ commander.countDown();
|
|
|
+ worker.await();
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error(e.getMessage());
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
}
|