update

src/main/java/cn/com/qmth/examcloud/tool/service/export_student_photo/ExportStudentPhotoTask.java

@@ -28,6 +28,9 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
 
 @Component
 public class ExportStudentPhotoTask implements TaskService {
@@ -52,14 +55,15 @@ public class ExportStudentPhotoTask implements TaskService {
     }
 
     private List<StudentVO> loadStudentFromUrl(User loginUser) {
+        long startTime = System.currentTimeMillis();
+
+        final String url = loginUser.getServerUrl() + "/api/ecs_core/student/studentPage/%s/%s?rootOrgId=%s&hasPhoto=TRUE";
         Map<String, String> headers = new HashMap<>();
         headers.put("key", loginUser.getKey());
         headers.put("token", loginUser.getToken());
 
+        long pageNo = 0, pageSize = 100, finishCount = 0, noPhotoCount = 0;
         JsonMapper jsonMapper = new JsonMapper();
-        long pageNo = 0, pageSize = 100, finishedCount = 0, noPhotoCount = 0;
-
-        final String url = loginUser.getServerUrl() + "/api/ecs_core/student/studentPage/%s/%s?rootOrgId=%s&hasPhoto=TRUE";
 
         List<StudentVO> hasPhotoStudents = new ArrayList<>();
         while (true) {
@@ -80,10 +84,12 @@ public class ExportStudentPhotoTask implements TaskService {
             }
 
             pageNo++;
-            long total = page.getTotal();
-            finishedCount += page.getList().size();
-            float finishedRate = finishedCount * 100f / total;
-            log.info("总数:{} 已获取学生数:{} 无底照学生数:{} pageNo:{} 进度:{}%", total, finishedCount, noPhotoCount, pageNo, finishedRate);
+            finishCount += page.getList().size();
+            float finishRate = finishCount * 100f / page.getTotal();
+            long cost = Math.max((System.currentTimeMillis() - startTime) / 1000, 1);
+            float speed = (float) finishCount / cost;
+            log.info("【获取学生】 总数:{} 已获取数:{} 无底照数:{} 进度:{}% 速度:约{}个每秒 已耗时:{}秒 pageNo:{}",
+                    page.getTotal(), finishCount, noPhotoCount, finishRate, speed, cost, pageNo);
         }
 
         return hasPhotoStudents;
@@ -95,6 +101,7 @@ public class ExportStudentPhotoTask implements TaskService {
             (select GROUP_CONCAT(DISTINCT sc.student_code) from ec_b_student_code sc where sc.student_id = stu.id)学号,stu.photo_path 照片
             from ec_b_student stu where stu.root_org_id = 0 and stu.photo_path is not null;
         */
+        long startTime = System.currentTimeMillis();
         List<StudentVO> list = new ArrayList<>();
         try {
             EasyExcel.read(dataFilePath, StudentVO.class, new AnalysisEventListener<StudentVO>() {
@@ -128,47 +135,80 @@ public class ExportStudentPhotoTask implements TaskService {
             hasPhotoStudents.add(student);
         }
 
-        log.info("总数:{} 有底照学生数:{} 无底照学生数:{}", list.size(), hasPhotoStudents.size(), noPhotoCount);
+        long cost = System.currentTimeMillis() - startTime;
+        log.info("【获取学生】 总数:{} 有底照数:{} 无底照数:{} 耗时:{}ms", list.size(), hasPhotoStudents.size(), noPhotoCount, cost);
         return hasPhotoStudents;
     }
 
     private void downloadStudents(List<StudentVO> students, String exportDir, boolean fileNameByStudentCode) {
+        long startTime = System.currentTimeMillis();
+        int totalCount = students.size();
+        AtomicInteger finishCount = new AtomicInteger(), successCount = new AtomicInteger(), failCount = new AtomicInteger();
+        ExecutorService executorService = Executors.newFixedThreadPool(5);
+
         for (StudentVO student : students) {
-            String photoSuffix = FileHelper.getFileSuffix(student.getPhotoPath());
-            if (fileNameByStudentCode) {
-                if (CollectionUtils.isNotEmpty(student.getStudentCodeList())) {
-                    if (student.getStudentCodeList().size() > 1) {
-                        log.warn("identityNumber:{} 多个学号！{}", student.getIdentityNumber(), student.getStudentCodesStr());
-                    }
-                    for (String studentCode : student.getStudentCodeList()) {
-                        String savePhotoPath = String.format("%s/%s/%s%s", exportDir, student.getRootOrgId(), studentCode, photoSuffix);
-                        this.downloadPhoto(student.getPhotoPath(), savePhotoPath);
+            executorService.execute(() -> {
+                String photoSuffix = FileHelper.getFileSuffix(student.getPhotoPath());
+                if (fileNameByStudentCode) {
+                    if (CollectionUtils.isNotEmpty(student.getStudentCodeList())) {
+                        if (student.getStudentCodeList().size() > 1) {
+                            log.warn("identityNumber:{} 多个学号！{}", student.getIdentityNumber(), student.getStudentCodesStr());
+                        }
+                        for (String studentCode : student.getStudentCodeList()) {
+                            String savePhotoPath = String.format("%s/%s/%s%s", exportDir, student.getRootOrgId(), studentCode, photoSuffix);
+                            this.downloadPhoto(student.getPhotoPath(), savePhotoPath, successCount, failCount);
+                        }
+                    } else {
+                        log.warn("identityNumber:{} 无学号！", student.getIdentityNumber());
+                        String savePhotoPath = String.format("%s/%s/%s%s", exportDir, student.getRootOrgId(), student.getIdentityNumber(), photoSuffix);
+                        this.downloadPhoto(student.getPhotoPath(), savePhotoPath, successCount, failCount);
                     }
                 } else {
-                    log.warn("identityNumber:{} 无学号！", student.getIdentityNumber());
                     String savePhotoPath = String.format("%s/%s/%s%s", exportDir, student.getRootOrgId(), student.getIdentityNumber(), photoSuffix);
-                    this.downloadPhoto(student.getPhotoPath(), savePhotoPath);
+                    this.downloadPhoto(student.getPhotoPath(), savePhotoPath, successCount, failCount);
                 }
-            } else {
-                String savePhotoPath = String.format("%s/%s/%s%s", exportDir, student.getRootOrgId(), student.getIdentityNumber(), photoSuffix);
-                this.downloadPhoto(student.getPhotoPath(), savePhotoPath);
-            }
+
+                finishCount.incrementAndGet();
+                if (finishCount.get() % 100 == 0) {
+                    float finishRate = finishCount.get() * 100f / totalCount;
+                    long cost = Math.max((System.currentTimeMillis() - startTime) / 1000, 1);
+                    float speed = (float) finishCount.get() / cost;
+                    log.info("【底照下载】 总数:{} 成功数:{} 失败数:{} 进度:{}% 速度:约{}个每秒 已耗时:{}秒",
+                            totalCount, successCount.get(), failCount.get(), finishRate, speed, cost);
+                }
+            });
         }
+
+        // 停止接受新任务，并等待所有任务完成
+        executorService.shutdown();
+        while (!executorService.isTerminated()) {
+            // ignore
+        }
+
+        float finishRate = finishCount.get() * 100f / totalCount;
+        long cost = Math.max((System.currentTimeMillis() - startTime) / 1000, 1);
+        float speed = (float) finishCount.get() / cost;
+        log.info("【底照下载】 总数:{} 成功数:{} 失败数:{} 进度:{}% 速度:约{}个每秒 已耗时:{}秒",
+                totalCount, successCount.get(), failCount.get(), finishRate, speed, cost);
     }
 
-    private void downloadPhoto(String photoUrl, String photoPath) {
-        String urlPrefix = "https://ecs-test-static.qmth.com.cn";
+    private void downloadPhoto(String photoUrl, String photoPath, AtomicInteger successCount, AtomicInteger failCount) {
+        // String urlPrefix = "https://ecs-test-static.qmth.com.cn";
+        String urlPrefix = "https://ecs-static.qmth.com.cn";
         photoUrl = FssHelper.finalFileUrl(photoUrl, urlPrefix);
 
         if (new File(photoPath).exists()) {
-            log.info("【已下载】 {} {}", photoPath, photoUrl);
+            log.debug("【已下载】 {} {}", photoPath, photoUrl);
+            successCount.incrementAndGet();
             return;
         }
 
         try {
             FileHelper.saveImageToFile(photoUrl, photoPath);
+            successCount.incrementAndGet();
         } catch (Exception e) {
-            log.error("【错误】 {} err:{}", photoPath, e.getMessage());
+            log.error("【下载错误】 {} err:{}", photoPath, e.getMessage());
+            failCount.incrementAndGet();
         }
     }