feat: p-queue优化队列

package.json

@@ -52,6 +52,7 @@
     "lodash": "^4.17.21",
     "mitt": "^3.0.0",
     "nprogress": "^0.2.0",
+    "p-queue": "^8.0.1",
     "pdfkit": "^0.15.0",
     "pg-hstore": "^2.3.4",
     "pinia": "^2.0.23",

pnpm-lock.yaml

@@ -58,6 +58,7 @@ specifiers:
   mitt: ^3.0.0
   mockjs: ^1.1.0
   nprogress: ^0.2.0
+  p-queue: ^8.0.1
   pdfkit: ^0.15.0
   pg-hstore: ^2.3.4
   pinia: ^2.0.23
@@ -97,6 +98,7 @@ dependencies:
   lodash: 4.17.21
   mitt: 3.0.1
   nprogress: 0.2.0
+  p-queue: 8.0.1
   pdfkit: 0.15.0
   pg-hstore: 2.3.4
   pinia: 2.1.7_pnzetbfa2uewunngbruulxbzye
@@ -4665,7 +4667,6 @@ packages:
 
   /eventemitter3/5.0.1:
     resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==}
-    dev: true
 
   /exec-buffer/3.2.0:
     resolution: {integrity: sha512-wsiD+2Tp6BWHoVv3B+5Dcx6E7u5zky+hUwOHjuH2hKSLR3dvRmX8fk8UD8uqQixHs4Wk6eDmiegVrMPjKj7wpA==}
@@ -7227,6 +7228,14 @@ packages:
     engines: {node: '>=8'}
     dev: true
 
+  /p-queue/8.0.1:
+    resolution: {integrity: sha512-NXzu9aQJTAzbBqOt2hwsR63ea7yvxJc0PwN/zobNAudYfb1B7R08SzB4TsLeSbUCuG467NhnoT0oO6w1qRO+BA==}
+    engines: {node: '>=18'}
+    dependencies:
+      eventemitter3: 5.0.1
+      p-timeout: 6.1.2
+    dev: false
+
   /p-reduce/1.0.0:
     resolution: {integrity: sha512-3Tx1T3oM1xO/Y8Gj0sWyE78EIJZ+t+aEmXUdvQgvGmSMri7aPTHoovbXEreWKkL5j21Er60XAWLTzKbAKYOujQ==}
     engines: {node: '>=4'}
@@ -7246,6 +7255,11 @@ packages:
       p-finally: 1.0.0
     dev: true
 
+  /p-timeout/6.1.2:
+    resolution: {integrity: sha512-UbD77BuZ9Bc9aABo74gfXhNvzC9Tx7SxtHSh1fxvx3jTLLYvmVhiQZZrJzqqU0jKbN32kb5VOKiLEQI/3bIjgQ==}
+    engines: {node: '>=14.16'}
+    dev: false
+
   /p-try/2.2.0:
     resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==}
     engines: {node: '>=6'}

src/views/base/track-export/usePQueue.ts

@@ -0,0 +1,36 @@
+import PQueue from 'p-queue';
+
+export type QueueFunction<T> = () => Promise<T>;
+
+export function usePQueue(option: {
+  concurrency?: number;
+  autoStart?: boolean;
+}) {
+  const queue = new PQueue(option);
+
+  async function buildQueue<T>(functions: QueueFunction<T>[]): Promise<T[]> {
+    try {
+      const results: T[] = await queue.addAll(functions);
+      return results;
+    } catch (error) {
+      console.error(error);
+      stopQueue();
+      throw error;
+    }
+  }
+
+  function startQueue() {
+    queue.start();
+  }
+
+  function stopQueue() {
+    queue.pause();
+    queue.clear();
+  }
+
+  return {
+    buildQueue,
+    startQueue,
+    stopQueue,
+  };
+}

src/views/base/track-export/useTask.pall.ts

@@ -0,0 +1,146 @@
+import { ref } from 'vue';
+import { trackExportDetailListPage, trackExportListPage } from '@/api/task';
+import {
+  TrackExportDetailItem,
+  TrackExportDetailListFilter,
+  TrackExportDetailListParams,
+  TrackExportItem,
+  TrackExportListFilter,
+  TrackExportListPageRes,
+} from '@/api/types/task';
+import {
+  TRACK_TASK_DETAIL_STATUS,
+  TRACK_TASK_STATUS,
+} from '../../../../electron/db/enumerate';
+import { TrackTaskCreationAttributes } from '../../../../electron/db/models/trackTask';
+
+export default function useTask() {
+  const pageSize = 20;
+  const trackTaskId = ref(0);
+
+  async function createTrackTask(data: TrackTaskCreationAttributes) {
+    const res = await window.db.createTrackTask(
+      { ...data, status: TRACK_TASK_STATUS.INIT },
+      []
+    );
+    trackTaskId.value = res.id;
+  }
+  async function updateTrackTaskReady() {
+    await window.db.updateTrackTaskStatus({
+      id: trackTaskId.value,
+      status: 'READY',
+    });
+  }
+
+  async function getTrackExportList(params: TrackExportListFilter) {
+    const trackExportList: TrackExportItem[] = [];
+    const res = await trackExportListPage({
+      ...params,
+      pageNumber: 1,
+      pageSize,
+    });
+    trackExportList.push(...res.records);
+
+    if (res.pages > 1) {
+      const fetchFunc: Promise<TrackExportListPageRes>[] = [];
+      for (let i = 2; i <= res.pages; i++) {
+        fetchFunc.push(
+          trackExportListPage({
+            ...params,
+            pageNumber: i,
+            pageSize,
+          })
+        );
+      }
+
+      const fetchResult = await Promise.all(fetchFunc);
+      fetchResult.forEach((item) => {
+        trackExportList.push(...item.records);
+      });
+    }
+
+    // get details
+    const detailFetchFunc = trackExportList.map((item) =>
+      getTrackExportDetailList({
+        examId: item.examId,
+        paperNumber: item.paperNumber,
+      })
+    );
+    await Promise.all(detailFetchFunc);
+  }
+
+  async function getTrackExportDetailList(params: TrackExportDetailListFilter) {
+    const filterData = {
+      ...params,
+      pageSize,
+    };
+    const res = await createTrackTaskDetails({
+      ...filterData,
+      pageNumber: 1,
+    });
+
+    if (res.pages <= 1) return;
+
+    const fetchFunc: Promise<any>[] = [];
+    for (let i = 2; i <= res.pages; i++) {
+      fetchFunc.push(
+        createTrackTaskDetails({
+          ...filterData,
+          pageNumber: i,
+        })
+      );
+    }
+
+    await Promise.all(fetchFunc);
+  }
+
+  async function createTrackTaskDetails(params: TrackExportDetailListParams) {
+    const res = await trackExportDetailListPage(params);
+    const details = res.records
+      .filter((item) => item.sheetUrls)
+      .map((item) => {
+        return {
+          trackTaskId: trackTaskId.value,
+          studentId: item.studentId,
+          studentName: item.studentName,
+          studentCode: item.studentCode,
+          className: item.className,
+          status: TRACK_TASK_DETAIL_STATUS.INIT,
+        };
+      });
+    await window.db.createTrackTaskDetails(details);
+
+    return {
+      total: res.total,
+      pages: res.pages,
+      pageNumber: params.pageNumber,
+    };
+  }
+
+  async function createTrackTaskDetailsFromStudents(
+    datas: TrackExportDetailItem[]
+  ) {
+    const details = datas
+      .filter((item) => item.sheetUrls)
+      .map((item) => {
+        return {
+          trackTaskId: trackTaskId.value,
+          studentId: item.studentId,
+          studentName: item.studentName,
+          studentCode: item.studentCode,
+          className: item.className,
+          status: TRACK_TASK_DETAIL_STATUS.INIT,
+        };
+      });
+    await window.db.createTrackTaskDetails(details);
+  }
+
+  return {
+    trackTaskId,
+    createTrackTask,
+    updateTrackTaskReady,
+    getTrackExportList,
+    getTrackExportDetailList,
+    createTrackTaskDetailsFromStudents,
+  };
+}

src/views/base/track-export/useTask.ts

@@ -13,6 +13,7 @@ import {
   TRACK_TASK_STATUS,
 } from '../../../../electron/db/enumerate';
 import { TrackTaskCreationAttributes } from '../../../../electron/db/models/trackTask';
+import { usePQueue } from './usePQueue';
 
 export default function useTask() {
   const pageSize = 20;
@@ -42,9 +43,9 @@ export default function useTask() {
     trackExportList.push(...res.records);
 
     if (res.pages > 1) {
-      const fetchFunc: Promise<TrackExportListPageRes>[] = [];
+      const fetchFuncs = [];
       for (let i = 2; i <= res.pages; i++) {
-        fetchFunc.push(
+        fetchFuncs.push(() =>
           trackExportListPage({
             ...params,
             pageNumber: i,
@@ -53,20 +54,29 @@ export default function useTask() {
         );
       }
 
-      const fetchResult = await Promise.all(fetchFunc);
+      const { buildQueue } = usePQueue({ concurrency: 6 });
+      const fetchResult = await buildQueue<TrackExportListPageRes>(fetchFuncs);
       fetchResult.forEach((item) => {
         trackExportList.push(...item.records);
       });
     }
 
     // get details
-    const detailFetchFunc = trackExportList.map((item) =>
-      getTrackExportDetailList({
-        examId: item.examId,
-        paperNumber: item.paperNumber,
-      })
-    );
-    await Promise.all(detailFetchFunc);
+    const detailFetchFuncs = trackExportList.map((item) => {
+      return () =>
+        getTrackExportDetailList({
+          examId: item.examId,
+          paperNumber: item.paperNumber,
+        });
+    });
+    const { buildQueue } = usePQueue({ concurrency: 1 });
+    await buildQueue(detailFetchFuncs);
+  }
+
+  interface CreateTrackTaskDetailsResult {
+    total: number;
+    pages: number;
+    pageNumber: number;
   }
 
   async function getTrackExportDetailList(params: TrackExportDetailListFilter) {
@@ -81,9 +91,9 @@ export default function useTask() {
 
     if (res.pages <= 1) return;
 
-    const fetchFunc: Promise<any>[] = [];
+    const fetchFuncs = [];
     for (let i = 2; i <= res.pages; i++) {
-      fetchFunc.push(
+      fetchFuncs.push(() =>
         createTrackTaskDetails({
           ...filterData,
           pageNumber: i,
@@ -91,7 +101,8 @@ export default function useTask() {
       );
     }
 
-    await Promise.all(fetchFunc);
+    const { buildQueue } = usePQueue({ concurrency: 6 });
+    await buildQueue<CreateTrackTaskDetailsResult>(fetchFuncs);
   }
 
   async function createTrackTaskDetails(params: TrackExportDetailListParams) {