注意！！！本次为addWatermarkBatch提交，发现批量处理优化作用甚微

src/features/ImageDownload/ImageDownload.vue

@@ -16,13 +16,14 @@
 
 <script setup lang="ts">
 import { store } from "@/store";
-import { onMounted, ref } from "vue";
+import { onMounted, ref, watch } from "vue";
 import { Modal } from "ant-design-vue";
 import { useRouter } from "vue-router";
 const router = useRouter();
 import { getStudents, countStudents, getPackages } from "@/api/api";
 import { httpApp } from "@/plugins/axiosApp";
 import mustache from "mustache";
+import { ImageParams } from "@/types";
 
 async function getImageDim(
   blob: Blob
@@ -46,7 +47,8 @@ async function cacheImages(urls: string) {
 
   async function sleep() {
     if (allPromiseCount - settledPromiseCount >= MAX_CONCURRENT) {
-      await new Promise((res) => setTimeout(res, 1000));
+      console.log("sleep because cache images thread is full");
+      await new Promise((res) => setTimeout(res, 300));
       await sleep();
     }
   }
@@ -67,6 +69,44 @@ async function cacheImages(urls: string) {
   }
 }
 
+const taskQueue = [] as Array<ImageParams>;
+// 批量处理水印
+async function batchWaterMarkImages() {
+  const MAX_CONCURRENT = 1;
+  console.log("start batchWaterMarkImages", taskQueue.length);
+
+  async function sleep() {
+    if (!doneFetch.value && taskQueue.length === 0) {
+      console.log("sleep because taskQueue is empty ", taskQueue.length);
+      await new Promise((res) => setTimeout(res, 300));
+      await sleep();
+    }
+  }
+
+  // 如果网络取完了，永不sleep
+  if (!doneFetch.value) {
+    await sleep();
+  }
+  const toProcessTasks = taskQueue.splice(0, MAX_CONCURRENT);
+  try {
+    console.log(toProcessTasks);
+    await window.electron.addWatermarkBatch(toProcessTasks);
+  } catch (error) {
+    console.log(error);
+  }
+
+  if (finished.value) {
+    console.log("被异常结束");
+  } else if (doneFetch.value && taskQueue.length === 0) {
+    finished.value = true;
+  } else {
+    console.log("继续batchWaterMarkImages ", taskQueue.length);
+    await batchWaterMarkImages();
+  }
+}
+
+let finished = ref(false); // 下载结束，无论有错误终止，还是完整下载完毕
+let doneFetch = ref(false); // 下载结束，无论有错误终止，还是完整下载完毕
 const totalCount = ref(0);
 let finishedCount = ref(0);
 let errorCount = ref(0);
@@ -74,8 +114,11 @@ let students = [];
 
 const config = store.pageInputs["/image-download"];
 onMounted(async () => {
+  const storePassedToNodeJs = JSON.parse(JSON.stringify(store));
   try {
     if (config.type === "1") {
+      console.log("download start ", Date.now());
+      batchWaterMarkImages();
       const res = await countStudents(store.env.examId, {
         upload: true,
         withSheetUrl: true,
@@ -122,7 +165,7 @@ onMounted(async () => {
                 config.dir,
                 mustache.render(config.template, student)
               );
-              if (window.electron.existsSync(filePath) && config.append) {
+              if (config.append && window.electron.existsSync(filePath)) {
                 console.log(filePath + " already exists");
                 // 执行到这里时，可能图片已经cache了
                 urls.splice(urls.indexOf(sheetUrl), 1);
@@ -135,26 +178,30 @@ onMounted(async () => {
               });
 
               const [width, height] = await getImageDim(imageRes.data);
+              const arrayBuffer = await imageRes.data.arrayBuffer();
 
               // console.log(imageRes.data);
               // console.log(await imageRes.data.arrayBuffer());
               // console.log(new Uint8Array(await imageRes.data.arrayBuffer()));
 
-              await window.electron.addWatermark(
-                JSON.parse(JSON.stringify(store)),
-                await imageRes.data.arrayBuffer(),
-                width,
-                height,
-                filePath,
+              taskQueue.push({
+                store: storePassedToNodeJs,
+                imageData: arrayBuffer,
+                imageWidth: width,
+                imageHeight: height,
+                file: filePath,
                 student,
-                index + 1,
-                config.trackMode,
-                config.x,
-                config.y
-              );
+                index: index + 1,
+                trackMode: config.trackMode,
+                x: config.x,
+                y: config.y,
+              });
+
+              // await window.electron.addWatermark();
             } catch (error) {
               errorCount.value += 1;
               if (config.failover) {
+                finished.value = true;
                 throw error;
               } else {
                 console.log(student, error);
@@ -162,65 +209,35 @@ onMounted(async () => {
               }
             }
           }
-          // 下载完一个学生
+          //   // 下载完一个学生
           finishedCount.value += 1;
-        }
-      }
-    } else if (config.type === "2") {
-      const res = await getPackages(store.env.examId, true, true);
-      const array = res.data;
-      totalCount.value = array.length;
-      const urls = array.reduce(
-        (accumulator, p) => accumulator.concat(p.urls),
-        []
-      );
-      cacheImages(urls);
-
-      for (let i = 0; i < array.length; i++) {
-        const p = array[i];
-        p.examId = store.env.examId;
-        for (let i = 0; i < p.urls.length; i++) {
-          try {
-            const index = i + 1;
-            p.index = index;
-            const filePath = window.electron.join(
-              config.dir,
-              mustache.render(config.template, p)
-            );
-            if (window.electron.existsSync(filePath) && config.append) {
-              console.log(filePath + " already exists");
-              urls.splice(urls.indexOf(p.urls[i]), 1);
-              continue;
-            }
 
-            const imageRes = await httpApp.get(p.urls[i], {
-              responseType: "blob",
-            });
-
-            await window.electron.saveImage(
-              await imageRes.data.arrayBuffer(),
-              filePath
+          // 等待照片水印处理，以免进度过快
+          const sleep = async () => {
+            console.log(
+              "sleep because network too fast,  taskQueue.length",
+              taskQueue.length
             );
-          } catch (error) {
-            errorCount.value += 1;
-            if (config.failover) {
-              throw error;
-            } else {
-              console.log(p, error);
-              continue;
+            if (taskQueue.length > 20) {
+              await new Promise((res) => setTimeout(res, 300));
+              await sleep();
             }
-          }
+          };
+          await sleep();
         }
-        finishedCount.value += 1;
       }
+      doneFetch.value = true;
+      console.log("network end ", Date.now());
+    } else if (config.type === "2") {
+      await processPackage();
     }
 
-    const modal = Modal.success({});
-    modal.update({
-      title: "图片下载完成",
-      content: "完成",
-      onOk: () => router.back(),
-    });
+    // const modal = Modal.success({});
+    // modal.update({
+    //   title: "图片下载完成",
+    //   content: "完成",
+    //   onOk: () => router.back(),
+    // });
   } catch (error) {
     const modal = Modal.error({});
     console.log(error);
@@ -231,4 +248,69 @@ onMounted(async () => {
     });
   }
 });
+
+// 只有config==1才会用到
+watch(finished, () => {
+  const modal = Modal.success({});
+  modal.update({
+    title: "图片下载完成",
+    content: "完成",
+    onOk: () => router.back(),
+  });
+  console.log("all end ", Date.now());
+});
+
+async function processPackage() {
+  const res = await getPackages(store.env.examId, true, true);
+  const array = res.data;
+  totalCount.value = array.length;
+  const urls = array.reduce((accumulator, p) => accumulator.concat(p.urls), []);
+  cacheImages(urls);
+
+  for (let i = 0; i < array.length; i++) {
+    const p = array[i];
+    p.examId = store.env.examId;
+    for (let i = 0; i < p.urls.length; i++) {
+      try {
+        const index = i + 1;
+        p.index = index;
+        const filePath = window.electron.join(
+          config.dir,
+          mustache.render(config.template, p)
+        );
+        if (config.append && window.electron.existsSync(filePath)) {
+          console.log(filePath + " already exists");
+          urls.splice(urls.indexOf(p.urls[i]), 1);
+          continue;
+        }
+
+        const imageRes = await httpApp.get(p.urls[i], {
+          responseType: "blob",
+        });
+
+        await window.electron.saveImage(
+          JSON.parse(JSON.stringify(store)),
+          await imageRes.data.arrayBuffer(),
+          filePath
+        );
+      } catch (error) {
+        errorCount.value += 1;
+        if (config.failover) {
+          throw error;
+        } else {
+          console.log(p, error);
+          continue;
+        }
+      }
+    }
+    finishedCount.value += 1;
+  }
+
+  const modal = Modal.success({});
+  modal.update({
+    title: "图片下载完成",
+    content: "完成",
+    onOk: () => router.back(),
+  });
+}
 </script>

src/lib/watermark.ts

@@ -2,12 +2,52 @@ import fs from "fs";
 import path from "path";
 // import sizeOf from "image-size";
 import mkdirp from "mkdirp";
-import { Store, Student } from "@/types";
+import { ImageParams, Store, Student } from "@/types";
 import gmType from "gm";
 
 let gm = null as unknown as typeof gmType;
 
-export async function addWatermark(
+export async function addWatermarkBatch(
+  imageArray: Array<ImageParams>
+): Promise<boolean> {
+  try {
+    for (const imageParams of imageArray) {
+      const {
+        store,
+        imageData,
+        imageWidth,
+        imageHeight,
+        file,
+        student,
+        index,
+        trackMode,
+        x,
+        y,
+      } = imageParams;
+      const startTime = Date.now();
+      await addWatermark(
+        store,
+        imageData,
+        imageWidth,
+        imageHeight,
+        file,
+        student,
+        index,
+        trackMode,
+        x,
+        y
+      );
+      console.log("watermarked one photo: ", Date.now() - startTime);
+    }
+
+    return true;
+  } catch (error) {
+    console.log(error);
+    return false;
+  }
+}
+
+async function addWatermark(
   store: Store,
   imageData: ArrayBuffer,
   imageWidth: number,
@@ -25,6 +65,30 @@ export async function addWatermark(
   //   store.config.watermark.color,
   //   student.objectiveScore
   // );
+  // console.log(
+  //   store,
+  //   imageData,
+  //   imageWidth,
+  //   imageHeight,
+  //   file,
+  //   student,
+  //   index,
+  //   trackMode,
+  //   x,
+  //   y
+  // );
+  if (
+    index !== 1 &&
+    (student.tags == undefined || student.tags[index] == undefined)
+  ) {
+    return await saveImage(store, imageData, file);
+  }
+
+  if (store.pageInputs["/image-download"].append && fs.existsSync(file)) {
+    console.log(file + " already exists");
+    return true;
+  }
+
   x = imageWidth * x;
   y = imageHeight * y;
   if (!gm) {
@@ -41,10 +105,6 @@ export async function addWatermark(
     // console.log(path.join(__dirname, "../../imagemagick/"));
     // console.log(path.join(__dirname, store.config.imagemagickDev));
   }
-  if (fs.existsSync(file) && store.pageInputs["/image-download"].append) {
-    console.log(file + " already exists");
-    return true;
-  }
   const fontFile = store.config.watermark.fontFile;
   const color = store.config.watermark.color;
   const image = Buffer.from(imageData);
@@ -256,9 +316,15 @@ export async function addWatermark(
 }
 
 export async function saveImage(
+  store: Store,
   imageData: ArrayBuffer,
   file: string
 ): Promise<boolean> {
+  if (store.pageInputs["/image-download"].append && fs.existsSync(file)) {
+    console.log(file + " already exists");
+    return true;
+  }
+
   return new Promise((resolve, reject) => {
     const image = Buffer.from(imageData);
     mkdirp.sync(path.dirname(file));

src/preload.ts

@@ -1,7 +1,7 @@
 import { contextBridge, remote } from "electron";
 
 import config from "./lib/config";
-import { addWatermark, saveImage } from "./lib/watermark";
+import { saveImage, addWatermarkBatch } from "./lib/watermark";
 import { queryStudentCount, replaceStudents, replacePackage } from "./lib/sync";
 import { existsSync } from "fs";
 import { join } from "path";
@@ -9,7 +9,8 @@ import { join } from "path";
 export const electronInWindow = {
   dialog: remote.dialog,
   config,
-  addWatermark,
+
+  addWatermarkBatch,
   saveImage,
   queryStudentCount,
   replaceStudents,

src/types/index.ts

@@ -122,3 +122,16 @@ export interface SubjectiveScoreDetail {
   groupNumber: number;
   mainTitle: string;
 }
+
+export interface ImageParams {
+  store: Store;
+  imageData: ArrayBuffer;
+  imageWidth: number;
+  imageHeight: number;
+  file: string;
+  student: Student;
+  index: number;
+  trackMode: string;
+  x?: number;
+  y?: number;
+}