优化实时人脸检测机制

src/features/OnlineExam/Examing/FaceTracking.vue

@@ -5,6 +5,7 @@ import { isThisMachineOwnByStudent } from "@/utils/utils";
 import { onMounted } from "vue";
 import { useTimers } from "@/setups/useTimers";
 import { store } from "@/store/store";
+import { throttle } from "lodash-es";
 
 const { addTimeout, addInterval } = useTimers();
 // window.faceapi = faceapi;
@@ -22,6 +23,35 @@ const { addTimeout, addInterval } = useTimers();
 //   };
 // })();
 
+onMounted(async () => {
+  await faceapi.nets.tinyFaceDetector.load(FACE_API_MODEL_PATH);
+  // faceapi.nets.faceRecognitionNet.load(modelsPath);
+  await faceapi.loadFaceLandmarkModel(FACE_API_MODEL_PATH);
+  faceapi.tf.ENV.set("WEBGL_PACK", false);
+
+  async function trackHead() {
+    const video = <HTMLVideoElement>document.getElementById("video");
+    if (video?.readyState === 4 && faceapi.nets.tinyFaceDetector.params) {
+      clearInterval(trackHeadInterval);
+    } else {
+      logger({
+        cnl: ["server"],
+        key: "FaceTracking",
+        act: "未达到实时人脸开启条件",
+      });
+      return;
+    }
+    logger({ cnl: ["server", "console"], act: "start tracking ... " });
+    await detectTest();
+
+    await detectFaces();
+  }
+
+  // 重复启动头部追踪，直到成功启动
+  const trackHeadInterval = addInterval(trackHead, 1000);
+});
+
+//#region webgl 参数
 let __cache4WebglAvailable: boolean | null = null;
 function webgl_available() {
   if (__cache4WebglAvailable !== null) return __cache4WebglAvailable;
@@ -46,24 +76,12 @@ function tensorFlowWebPackStatus() {
   }
   return __cache4TensorFlowWebPackStatus;
 }
+//#endregion
 
-// function getCPUModel() {
-//   if (typeof nodeRequire != "undefined") {
-//     var os = window.nodeRequire("os");
-//     const cpus = os.cpus();
-//     if (cpus.length > 0) {
-//       return cpus[0].model;
-//     }
-//   }
-//   return "null";
-// }
-
-// if (os.isWin7) alert("是win7");
-// if (os.isWin10) alert("是win10");
-
-let __inputSize = 128;
+let bestInputSize = 128;
 let disableFaceTracking = false;
 
+/** 测试学生电脑适合的参数 */
 async function detectTest() {
   const inputSizeList = [128, 160, 224, 320, 416, 512, 608];
   const succRate = [0, 0, 0, 0, 0, 0, 0];
@@ -82,11 +100,7 @@ async function detectTest() {
       new Promise((resolve) => setTimeout(resolve, 10 * 1000)),
     ]);
     const detectEndTime = performance.now();
-    if (
-      !result ||
-      !result.length ||
-      detectStartTime - detectEndTime > 2 * 1000
-    ) {
+    if (!result || detectEndTime - detectStartTime > 2 * 1000) {
       disableFaceTracking = true;
       _hmt.push(["_trackEvent", "答题页面", "启动检测耗时过长：停止实时"]);
       logger({
@@ -95,13 +109,12 @@ async function detectTest() {
         dtl: "启动检测耗时过长：停止实时",
         ext: {
           result: JSON.stringify(result),
-          cost: detectStartTime - detectEndTime,
+          cost: detectEndTime - detectStartTime,
         },
       });
       return;
     }
   } catch (error) {
-    console.log(error);
     disableFaceTracking = true;
     _hmt.push(["_trackEvent", "答题页面", "启动检测错误：停止实时"]);
     logger({
@@ -116,7 +129,7 @@ async function detectTest() {
     for (let n = 0; n < detectTimes; n++) {
       await new Promise((resolve) => setTimeout(resolve, 3 * 1000));
       if (store.exam.isDoingFaceLiveness) {
-        console.log("正在活检，暂停实时人脸");
+        logger({ cnl: ["server", "console"], act: "正在活检，暂停实时人脸" });
         await new Promise((resolve) => setTimeout(resolve, 120 * 1000));
       }
       const inputSize = inputSizeList[idx];
@@ -149,41 +162,60 @@ async function detectTest() {
         }
 
         if (result && result.length >= 1) {
-          console.log(`inputSize: ${inputSize} ${result.length}`);
+          logger({
+            cnl: ["server"],
+            key: "FaceTracking",
+            dtl: `inputSize: ${inputSize} ${result.length}`,
+          });
           succRate[idx]++;
         } else {
-          console.log(`inputSize: ${inputSize} 检测失败`);
+          logger({
+            cnl: ["server"],
+            key: "FaceTracking",
+            act: "FT检测失败",
+            dtl: `inputSize: ${inputSize}`,
+          });
         }
       } catch (error) {
-        console.log(error);
-        console.log(`inputSize: ${inputSize} 检测失败-异常`);
+        logger({
+          cnl: ["server"],
+          key: "FaceTracking",
+          act: "FT检测失败-异常",
+          dtl: `inputSize: ${inputSize}`,
+          possibleError: error,
+        });
       }
     }
 
     if (succRate[idx] === detectTimes) {
-      console.log(`inputSize: ${inputSizeList[idx]} 提前选中`);
+      logger({
+        cnl: ["server"],
+        key: "FaceTracking",
+        act: "FT提前选中",
+        dtl: `inputSize: ${inputSizeList[idx]}`,
+      });
       break;
     }
   }
 
-  console.log({ succRate });
   const max = Math.max(...succRate);
 
   const idx = succRate.indexOf(max);
 
-  __inputSize = inputSizeList[idx];
+  bestInputSize = inputSizeList[idx];
   logger({
     cnl: ["server", "local"],
     pgn: "实时人脸检测",
-    act: "最好的 inputSize 为:" + __inputSize,
+    dtl: "最好的 inputSize 为:" + bestInputSize,
+    ext: { succRate },
   });
 
-  return __inputSize;
+  return bestInputSize;
 }
 
 function getFaceDetectorOptions() {
   return new faceapi.TinyFaceDetectorOptions({
-    inputSize: __inputSize || 128,
+    inputSize: bestInputSize || 128,
     scoreThreshold: 0.5,
   });
 
@@ -191,53 +223,23 @@ function getFaceDetectorOptions() {
   // return new faceapi.MtcnnOptions({ minFaceSize: 200, scaleFactor: 0.8 });
 }
 
-const detectTimeArray: number[] = [];
-
-onMounted(async () => {
-  await faceapi.nets.tinyFaceDetector.load(FACE_API_MODEL_PATH);
-  // faceapi.nets.faceRecognitionNet.load(modelsPath);
-  await faceapi.loadFaceLandmarkModel(FACE_API_MODEL_PATH);
-  faceapi.tf.ENV.set("WEBGL_PACK", false);
-
-  let trackStarted = false;
-
-  async function trackHead() {
-    const video = <HTMLVideoElement>document.getElementById("video");
-    if (
-      video &&
-      video.readyState === 4 &&
-      faceapi.nets.tinyFaceDetector.params
-    ) {
-      trackStarted = true;
-    } else {
-      return;
-    }
-    console.log("start tracking ... ");
-    await detectTest();
-
-    await detectFaces();
-  }
-  const trackHeadInterval = addInterval(() => {
-    if (trackStarted) {
-      clearInterval(trackHeadInterval);
-    } else {
-      void trackHead();
-    }
-  }, 1000);
-});
-// beforeDestroy() {
-//   clearTimeout(this.warningTimeout);
-//   clearTimeout(this.detectFacesTimeout);
-// },
+const indepentExamingMsg = throttle(
+  () => $message.warning("请独立完成考试"),
+  20 * 1000
+);
+const posureExamingMsg = throttle(
+  () => $message.warning("请调整坐姿，诚信考试"),
+  20 * 1000
+);
 
 let singleTimeUsage = 0;
 let multipleTimeUsage = 0;
-let showWaringTime = Date.now();
-
+const detectTimeArray: number[] = [];
 let failTimes = 0;
 
 let detectFacesTimeout: number;
-let warningTimeout: number;
+
+/** 定时检测人脸 */
 async function detectFaces() {
   if (
     disableFaceTracking ||
@@ -252,34 +254,25 @@ async function detectFaces() {
     _hmt.push(["_trackEvent", "答题页面", "关闭实时人脸检测，因为耗时过长"]);
     return;
   }
-  // FIXME: 接收活体进行中的事件
-  // if (this.isDoingFaceLiveness) {
-  //   logger({
-  //     cnl: ["server"],
-  //     pgn: "实时人脸检测",
-  //     act: "正在活检，暂停实时人脸",
-  //   });
-  //   clearTimeout(detectFacesTimeout);
-  //   detectFacesTimeout = addTimeout(() => void detectFaces(), 10 * 1000);
-  //   return;
-  // }
+  if (store.exam.isDoingFaceLiveness) {
+    logger({
+      cnl: ["server"],
+      pgn: "实时人脸检测",
+      act: "正在活检，暂停实时人脸",
+    });
+    clearTimeout(detectFacesTimeout);
+    detectFacesTimeout = addTimeout(() => void detectFaces(), 10 * 1000);
+    return;
+  }
 
   const videoEl = <HTMLVideoElement>document.getElementById("video");
-  // var canvas = document.createElement("canvas");
-  // canvas.width = 133;
-  // canvas.height = 100;
-
-  // var context = canvas.getContext("2d");
-  // context.drawImage(videoEl, 0, 0, 133, 100);
   const detectStartTime = performance.now();
-  // this.___vWidth =
-  //   this.___vWidth ||
-  //   document.getElementById("video-container").clientWidth;
 
   const options = getFaceDetectorOptions();
   let result;
 
   try {
+    logger({ cnl: ["server"], key: "FaceTracking", act: "开始一次人脸检测" });
     result = await faceapi
       // .detectSingleFace(videoEl, options)
       .detectAllFaces(videoEl, options);
@@ -288,17 +281,34 @@ async function detectFaces() {
     logger({ cnl: ["server"], act: "实时人脸检测失败", possibleError: e });
     throw e;
   }
-  // console.log(result);
+
+  if (!result) {
+    try {
+      logger({
+        cnl: ["server"],
+        key: "不可能的事情发生了",
+        dtl: "人脸检测结果格式不符合预期",
+        stk: JSON.stringify(result || {}),
+      });
+    } catch (error) {
+      logger({
+        cnl: ["server"],
+        key: "不可能的事情发生了",
+        dtl: "人脸检测结果stringify错误",
+      });
+    }
+  }
 
   const detectEndTime = performance.now();
   logger({
     cnl: ["server", "console"],
     pgn: "实时人脸检测",
+    act: "做完一次人脸检测，准备统计...",
     ext: {
+      resultLen: result.length,
       WebGL: webgl_available(),
       WEBGL_PACK: tensorFlowWebPackStatus(),
       "single detect time": detectEndTime - detectStartTime,
-      resultLen: result.length,
     },
   });
   singleTimeUsage = detectEndTime - detectStartTime;
@@ -316,11 +326,11 @@ async function detectFaces() {
       cnl: ["server"],
       pgn: "实时人脸检测",
       ext: {
+        detectTimeArray,
         roundAvg: roundAvg + "ms",
         computer: isThisMachineOwnByStudent() ? "学生电脑" : "学习中心电脑",
       },
     });
-    console.log(detectTimeArray);
     detectTimeArray.push(0, 0); // 避免再次达到push条件和上传条件
 
     // FIXME: 上线初期停止统计此类信息，过于零散
@@ -336,40 +346,31 @@ async function detectFaces() {
 
     multipleTimeUsage = roundAvg;
   }
-  // init this.showWaringTime
-  showWaringTime = showWaringTime || Date.now();
 
-  if (result.length >= 2 && Date.now() - showWaringTime > 20 * 1000) {
-    showWaringTime = Date.now();
-    $message.warning("请独立完成考试");
+  if (result.length >= 2) {
+    indepentExamingMsg();
   }
 
-  if (result.length === 0 && Date.now() - showWaringTime > 20 * 1000) {
-    showWaringTime = Date.now();
-    $message.warning("请调整坐姿，诚信考试");
-    failTimes = failTimes || failTimes++;
+  if (result.length === 0) {
+    posureExamingMsg();
+    failTimes++;
   }
 
-  if (
-    (!result || result.length !== 1) &&
-    !videoEl.classList.contains("video-warning")
-  ) {
+  if (result.length !== 1 && !videoEl.classList.contains("video-warning")) {
     videoEl.classList.add("video-warning");
-    clearTimeout(warningTimeout);
-    warningTimeout = addTimeout(function () {
-      videoEl.classList.remove("video-warning");
-    }, 3000);
+    addTimeout(() => videoEl.classList.remove("video-warning"), 3000);
   }
 
   clearTimeout(detectFacesTimeout);
+  logger({ cnl: ["server"], lvl: "debug", act: "准备下次人脸检测" });
   detectFacesTimeout = addTimeout(async () => {
-    if (failTimes > 5) {
+    if (failTimes >= 5) {
       $message.warning("请保持正确坐姿，确保脸部在摄像头内，背景无强光。");
-      failTimes = 1;
+      failTimes = 0;
       await detectTest();
     }
     await detectFaces();
-  }, 60 * 1000);
+  }, 20 * 1000);
 }
 </script>
 

src/plugins/axiosNotice.ts

@@ -2,9 +2,7 @@ import { throttle } from "lodash-es";
 import { showLogout } from "@/utils/utils";
 
 export const notifyInvalidTokenThrottled = throttle(
-  () => {
-    showLogout("登录失效");
-  },
+  () => showLogout("登录失效"),
   3000,
   { trailing: false }
 );

src/utils/logger.ts

@@ -100,6 +100,10 @@ export default function createLog(detail: LogDetail) {
   if (import.meta.env.PROD && newDetail.lvl !== "log") {
     return;
   }
+  // 开发阶段将日志全部打印出来
+  if (import.meta.env.DEV && !detail.cnl.includes("console")) {
+    detail.cnl.push("console");
+  }
   if (detail.cnl?.includes("console")) {
     if (import.meta.env.DEV) {
       console.log(