feat: 数据批量保存优化

src/mixins/initStoreMixin.js

@@ -19,11 +19,10 @@ export default {
         "domain",
         "ocrArea",
       ];
-      for (let i = 0; i < needInitDict.length; i++) {
-        if (!Object.prototype.hasOwnProperty.call(storeDict, needInitDict[i])) {
-          await db.initDict(needInitDict[i]);
-        }
-      }
+      const datas = needInitDict
+        .filter((key) => !Object.prototype.hasOwnProperty.call(storeDict, key))
+        .map((key) => [key, ""]);
+      await db.batchSaveDict(datas);
 
       const domain = storeDict["domain"] || "";
       this.$store.commit("setDomain", domain);

src/modules/client/views/ScanOther.vue

@@ -79,7 +79,7 @@
 <script>
 import {
   getPreUploadFiles,
-  saveOutputImage,
+  batchSaveImages,
   clearDir,
   getDirScanFile,
   getScanFileBasename,
@@ -281,7 +281,7 @@ export default {
           fileTypeId: this.modalForm.fileTypeId,
           fileTypeName: this.modalForm.fileTypeName,
           roomOrClass: this.modalForm.roomOrClass,
-          batchNo: "",
+          batchNo: "other",
           clientUserId: this.user.id,
           clientUsername: this.user.loginName,
           clientUserLoginTime: this.user.loginTime,
@@ -290,19 +290,6 @@ export default {
         this.scanStageList.push(fileInfo);
       }
     },
-    async saveScanItem(fileInfo) {
-      const ouputImageList = saveOutputImage(
-        [fileInfo.frontOriginImgPath, fileInfo.versoOriginImgPath],
-        {
-          courseCode: this.task.courseCode,
-        }
-      );
-
-      fileInfo.frontOriginImgPath = ouputImageList[0];
-      fileInfo.versoOriginImgPath = ouputImageList[1];
-
-      await db.saveUploadInfo(fileInfo);
-    },
     async toSave() {
       if (!this.scanStageList.length) {
         this.$message.error("当前无要保存的数据！");
@@ -312,23 +299,19 @@ export default {
       if (this.saving) return;
       this.saving = true;
 
-      // TODO: 使用批量保存，有时间再做
       logger.info(`04-1开始保存数据`);
-      for (let i = 0, len = this.scanStageList.length; i < len; i++) {
-        const fileInfo = this.scanStageList[i];
-        fileInfo.batchNo = "other";
-
-        let res = true;
-        await this.saveScanItem(fileInfo).catch((err) => {
-          res = false;
-          console.error(err);
-          logger.error(`04-1保存数据错误，${err}`);
-        });
-        if (!res) {
-          this.saving = false;
-          this.$message.error("保存数据错误，请重新尝试！");
-          return Promise.reject();
-        }
+      try {
+        this.scanStageList = await batchSaveImages(
+          this.scanStageList,
+          this.task.courseCode
+        );
+        await db.batchSaveUploadInfo(this.scanStageList);
+      } catch (err) {
+        console.error(err);
+        logger.error(`04-1保存数据错误，${err}`);
+        this.saving = false;
+        this.$message.error("保存数据错误，请重新尝试！");
+        return Promise.reject();
       }
 
       this.$message.success("保存成功！");

src/modules/client/views/ScanPaper.vue

@@ -104,11 +104,11 @@
 import { mapState } from "vuex";
 import {
   getPreUploadFiles,
-  saveOutputImage,
   clearDir,
   deleteFiles,
   decodeImageCode,
   getDirScanFile,
+  batchSaveImages,
 } from "../../../plugins/imageOcr";
 import db from "../../../plugins/db";
 import { evokeScanner } from "../../../plugins/scanner";
@@ -414,19 +414,6 @@ export default {
         this.setCurPaper(0);
       }
     },
-    async saveScanItem(fileInfo) {
-      const ouputImageList = saveOutputImage(
-        [fileInfo.frontOriginImgPath, fileInfo.versoOriginImgPath],
-        {
-          courseCode: this.task.courseCode,
-        }
-      );
-
-      fileInfo.frontOriginImgPath = ouputImageList[0];
-      fileInfo.versoOriginImgPath = ouputImageList[1];
-
-      await db.saveUploadInfo(fileInfo);
-    },
     toSave() {
       if (this.errorStageList.length) {
         this.$message.error("还有异常数据未处理！");
@@ -443,24 +430,24 @@ export default {
       if (this.saving) return;
       this.saving = true;
 
-      const datas = this.scanStageList.map((item) => item.papers).flat();
-      // TODO: 使用批量保存，有时间再做
+      this.clearViewPapers();
       logger.info(`04-1开始保存数据`);
-      for (let i = 0, len = datas.length; i < len; i++) {
-        const fileInfo = datas[i];
-        fileInfo.batchNo = batchNo;
-
-        let res = true;
-        await this.saveScanItem(fileInfo).catch((err) => {
-          res = false;
-          console.error(err);
-          logger.error(`04-1保存数据错误，${err}`);
-        });
-        if (!res) {
-          this.saving = false;
-          this.$message.error("保存数据错误，请重新尝试！");
-          return Promise.reject();
-        }
+      try {
+        let datas = this.scanStageList
+          .map((item) => item.papers)
+          .flat()
+          .map((item) => {
+            item.batchNo = batchNo;
+            return item;
+          });
+        datas = await batchSaveImages(datas, this.task.courseCode);
+        await db.batchSaveUploadInfo(datas);
+      } catch (err) {
+        console.error(err);
+        logger.error(`04-1保存数据错误，${err}`);
+        this.saving = false;
+        this.$message.error("保存数据错误，请重新尝试！");
+        return Promise.reject();
       }
 
       this.$message.success("保存成功！");
@@ -474,9 +461,9 @@ export default {
     async loopScaningFile() {
       this.clearSetTs();
       if (!this.looping) return;
-      const st = Date.now();
+      // const st = Date.now();
       await this.getScaningFile();
-      console.log(`耗时：${Date.now() - st}ms`);
+      // console.log(`耗时：${Date.now() - st}ms`);
 
       this.addSetTime(this.loopScaningFile, 1 * 1000);
     },

src/plugins/db.js

@@ -94,7 +94,34 @@ function saveUploadInfo(params) {
   });
 }
 
-function batchSaveUploadInfo(datas) {
+function batchSaveUploadInfo(dataList) {
+  const createdTime = formatDate();
+  const datas = dataList.map((item) => {
+    return [
+      item.taskId,
+      item.schoolId,
+      item.semesterId,
+      item.examId,
+      item.courseCode,
+      item.courseName,
+      item.frontOriginImgPath,
+      item.versoOriginImgPath,
+      item.studentCode,
+      item.ocrArea,
+      item.fileTypeId,
+      item.fileTypeName,
+      item.roomOrClass,
+      item.batchNo,
+      item.isFormal,
+      item.clientUserId,
+      item.clientUsername,
+      item.clientUserLoginTime,
+      0, // isUpload
+      createdTime, // createdTime
+      null, // finishTime
+    ];
+  });
+
   return new Promise((resolve, reject) => {
     const sql = `INSERT INTO scan (taskId, schoolId, semesterId, examId, courseCode, courseName, frontOriginImgPath, versoOriginImgPath, studentCode, ocrArea,fileTypeId,fileTypeName,roomOrClass,batchNo, isFormal, clientUserId, clientUsername, clientUserLoginTime, isUpload,createdTime, finishTime) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`;
     db.serialize(() => {
@@ -243,6 +270,23 @@ function setDict(key, val) {
   });
 }
 
+function batchSaveDict(datas) {
+  return new Promise((resolve, reject) => {
+    const sql = `INSERT INTO dict (key, val) VALUES (?,?)`;
+    db.serialize(() => {
+      const stmt = db.prepare(sql);
+
+      for (let i = 0; i < datas.length; i++) {
+        stmt.run(datas[i]);
+      }
+      stmt.finalize((err) => {
+        if (err) reject(err);
+        resolve();
+      });
+    });
+  });
+}
+
 export default {
   init,
   // scan
@@ -259,4 +303,5 @@ export default {
   setDict,
   getAllDict,
   initDict,
+  batchSaveDict,
 };

src/plugins/imageOcr.js

@@ -7,6 +7,7 @@ import {
   getTmpDir,
 } from "./env";
 import { randomCode, formatDate } from "./utils";
+import usePQueue from "../plugins/pQueue";
 import gm from "./gm";
 const fs = window.nodeRequire("fs");
 const path = window.nodeRequire("path");
@@ -307,3 +308,45 @@ export function decodeImageCode(imgPath, codeArea) {
     });
   });
 }
+
+export async function batchSaveImages(imageList, courseCode) {
+  const outputDir = path.join(getOutputDir("origin"), courseCode);
+  if (!fs.existsSync(outputDir)) makeDirSync(outputDir);
+
+  const { buildQueue } = usePQueue({ concurrency: 6 });
+
+  const images = imageList
+    .map((item, index) => {
+      return [
+        {
+          index,
+          front: true,
+          url: item.frontOriginImgPath,
+        },
+        {
+          index,
+          front: false,
+          url: item.versoOriginImgPath,
+        },
+      ];
+    })
+    .flat();
+
+  const tasks = images.map((item) => {
+    return async () => {
+      const outputOriginPath = path.join(outputDir, path.basename(item.url));
+      let result = true;
+      await fs.promises.rename(item.url, outputOriginPath).catch((err) => {
+        console.error(`Error moving file: ${item.url}, ${err.message}`);
+        result = false;
+      });
+      if (!result) return;
+      const urlKey = item.front ? "frontOriginImgPath" : "versoOriginImgPath";
+      imageList[item.index][urlKey] = outputOriginPath;
+    };
+  });
+
+  await buildQueue(tasks);
+
+  return imageList;
+}