student-client-v3/src/features/OnlineExam/FaceRecognition.vue

<script setup lang="ts">
import MD5 from "js-md5";
import { onMounted, watchEffect } from "vue";
import { getMediaStream } from "@/utils/camera";
import { httpApp } from "@/plugins/axiosApp";
import { showLogout } from "@/utils/utils";
import { getCapturePhotoYunSign, saveCapturePhoto } from "@/api/login";
import { execLocal, fileExists } from "@/utils/nativeMethods";

/**
 * 上层通过showRecognizeButton来控制是否是同步比对
 *
 * 同步比对通过onRecognizeResult得到人脸比对结果
 *
 * 异步比对通过snapId来控制是否该进行比对，什么时候进行，以什么频率频率进行，错误处理，均由上层控制
 * 异步比对同时传递一个snapId(time)，供上层识别和计数
 * 可能存在多个异步比对的任务同时进行
 */

const {
  width = 400,
  height = 300,
  snapId = 0,
  examRecordDataId = -1,
  showRecognizeButton = false,
} = defineProps<{
  width: string;
  height: string;
  showRecognizeButton: boolean;
  snapId?: number;
  examRecordDataId?: number;
}>();

const emit = defineEmits<{
  (
    e: "on-recognize-result",
    v: { isPassed: boolean; isStranger: boolean }
  ): void;
  (
    e: "on-async-recognize-result",
    v: { hasError: boolean; fileName: string }
  ): void;
}>();

let snapBtnDisabled = $ref(true);
let btnText = $ref("开始识别");

watchEffect(() => {
  if (snapId) {
    void snapAsync();
  }
});

onMounted(async () => {
  await openCamera();
});

const video = $ref<HTMLVideoElement>();

async function openCamera() {
  const _openStartTime = Date.now();
  const stream = await getMediaStream();
  video.srcObject = stream;
  try {
    await video.play();
  } catch (error) {
    if (error instanceof Error) {
      if (error.name == "AbortError") {
        logger({
          cnl: ["server"],
          act: "video.paly",
          dtl: "AbortError and retry",
        });
        await video.play();
        logger({
          cnl: ["server"],
          act: "摄像头没有正常启用: AbortError 重试成功",
        });
      } else if (error.name == "NotSupportedError") {
        logger({
          cnl: ["server"],
          act: "摄像头没有正常启用",
          ejn: JSON.stringify(error),
          ext: {
            errorName: error.name,
            errorMessage: error.message,
            errorStack: error.stack,
          },
        });
        $message.error("摄像头没有正常启用: " + error);
      } else {
        throw error;
      }
    } else {
      logger({
        cnl: ["server"],
        act: "video.play",
        dtl: "not an Error",
        stk: error + "",
      });
    }
    throw error;
  }
  snapBtnDisabled = false;
  const _openEndTime = Date.now();
  logger({
    cnl: ["server"],
    act: "摄像头打开耗时",
    ext: { cost: _openEndTime - _openStartTime },
  });
}

async function videoStartPlay() {
  if (video && video.paused) {
    await video.play().catch((e) => {
      if (!(e instanceof Error)) {
        logger({
          cnl: ["server"],
          act: "restart video play error",
          dtl: "not an Error",
          stk: e + "",
        });
      } else {
        logger({
          cnl: ["server"],
          act: "restart video play error",
          stk: e.stack,
          ejn: JSON.stringify(e),
        });
      }
      throw e;
    });
  }
}

//#region 同步人脸比对
async function snapSync() {
  logger({
    cnl: ["server"],
    act: "同步人脸比对",
    dtl: "点击开始识别按钮",
  });

  $message.destroyAll();
  try {
    snapBtnDisabled = true;
    btnText = "拍照中...";
    logger({ cnl: ["server"], lvl: "debug", act: btnText });
    const captureBlob = await getSnapShot(true);
    if (!(captureBlob instanceof Blob)) return;

    logger({
      cnl: ["server"],
      lvl: "debug",
      act: "getSnapShot",
      ext: { blobSize: captureBlob.size },
    });
    if (captureBlob.size < 48 * 48 || captureBlob.size >= 2 * 1024 * 1024) {
      $message.error("抓拍照片太小！");
      logger({
        cnl: ["server"],
        act: "摄像头异常",
        dtl: "抓拍照片大小异常",
        ext: { blobSize: captureBlob.size },
      });
      throw new Error("抓拍照片大小异常");
    }
    btnText = "上传照片中...";
    logger({ cnl: ["server"], lvl: "debug", act: btnText });
    const [captureFilePath, signIdentifier] = await uploadToServer(captureBlob);
    btnText = "人脸比对中...";
    await faceCompareSync(captureFilePath, signIdentifier);
    logger({
      cnl: ["server"],
      act: "同步比对照片详细日志",
      ext: {
        fileUrl: captureFilePath,
        signIdentifier,
      },
    });
  } catch (error) {
    console.log("同步照片比对流程失败");
    throw error;
  } finally {
    btnText = "开始识别";
    // 避免人脸识别功能被大量重复点击
    await new Promise((resolve) => setTimeout(resolve, 3000));
    snapBtnDisabled = false;
  }
}
async function getSnapShot(compareSync: boolean): Promise<Blob | unknown> {
  return new Promise((resolve, reject) => {
    if (video.readyState !== 4 || !(video.srcObject as MediaStream).active) {
      $message.error("摄像头没有正常启用");
      logger({
        cnl: ["server"],
        pgu: "AUTO",
        act: "getSnapShot",
        dtl: "摄像头没有正常启用",
      });
      reject("摄像头没有正常启用");
      if (!compareSync) {
        showLogout("摄像头没有正常启用");
      }
      return;
    }
    // video.pause();
    const canvas = document.createElement("canvas");
    canvas.width = 220;
    canvas.height = 165;

    const context = canvas.getContext("2d");
    context?.drawImage(video, 0, 0, 220, 165);

    canvas.toBlob((blob) => resolve(blob!), "image/png", 0.95);
  })
  // .finally(() => void videoStartPlay()); // finally 会在返回前执行，满足我们的要求
}

// 用来比对两次抓拍照片的md5是否一样
let __previousPhotoMD5 = "";
async function uploadToServer(captureBlob: Blob): Promise<[string, string]> {
  async function blobToArray(blob: Blob): Promise<ArrayBuffer> {
    return new Promise((resolve) => {
      var reader = new FileReader();
      reader.addEventListener("loadend", function () {
        // reader.result contains the contents of blob as a typed array
        resolve(reader.result as ArrayBuffer);
      });
      reader.readAsArrayBuffer(blob);
    });
  }
  //保存抓拍照片到服务器
  let resultUrl, signIdentifier;
  try {
    const buffer = await blobToArray(captureBlob);

    const fileMd5Base64 = window.btoa(
      String.fromCharCode(...MD5.digest(buffer))
    );
    if (fileMd5Base64 === __previousPhotoMD5) {
      logger({
        cnl: ["server"],
        pgu: "AUTO",
        act: "uploadToServer",
        key: "抓拍照片异常",
        stk: "两次fileMd5Base64一样，疑似摄像头卡住",
      });
    }
    __previousPhotoMD5 = fileMd5Base64;

    const res = await getCapturePhotoYunSign({ fileSuffix: "png" });

    try {
      const saveRes = await saveCapturePhoto(
        res.data.formUrl,
        res.data.formParams,
        { file: captureBlob }
      );
      if (saveRes.headers["content-md5"] != fileMd5Base64) {
        logger({
          cnl: ["server"],
          dtl: "抓拍照片保存失败--alioss content-md5 mismatch",
          ext: {
            fileMd5Base64,
            "content-md5": saveRes.headers["content-md5"],
          },
        });
        throw new Error("图片校验失败");
      }
    } catch (error) {
      logger({
        cnl: ["server"],
        dtl: "抓拍照片保存失败",
        possibleError: error,
      });
      throw error;
    }

    // console.log(response);
    resultUrl = res.data.accessUrl;
    signIdentifier = res.data.signIdentifier;
    logger({
      cnl: ["server"],
      dtl: "抓拍照片保存成功",
      ext: { resultUrl, fileMd5Base64 },
    });
  } catch (e) {
    console.log(e);
    logger({
      cnl: ["server"],
      pgu: "AUTO",
      dtl: "保存抓拍照片到服务器失败！",
      possibleError: e,
    });
    $message.error("抓拍照片保存失败！");
    throw new Error("抓拍照片保存失败！");
  }

  return [resultUrl, signIdentifier];
}

async function faceCompareSync(
  captureFilePath: string,
  signIdentifier: string
) {
  try {
    logger({ cnl: ["server"], act: "同步比对开始" });
    const res = await httpApp.post(
      "/api/ecs_oe_student_face/examCaptureQueue/compareFaceSync?signIdentifier=" +
        signIdentifier +
        "&fileUrl=" +
        encodeURIComponent(captureFilePath)
    );
    logger({
      cnl: ["server"],
      act: "同步比对api成功",
      ext: {
        isPass: res.data.isPass,
        isStranger: res.data.isStranger,
        errorMsg: res.data.errorMsg,
      },
    });
    emit("on-recognize-result", {
      isPassed: res.data.isPass,
      isStranger: res.data.isStranger,
    });
  } catch (e) {
    logger({
      cnl: ["server"],
      act: "同步比对失败",
      possibleError: e,
    });
    throw new Error("同步照片比较失败！");
  }
}
//#endregion 同步人脸比对

//#region 异步人脸比对
async function snapAsync() {
  try {
    logger({ cnl: ["server"], act: "定时抓拍开始" });
    const captureBlob = await getSnapShot(false);
    if (!(captureBlob instanceof Blob)) return;

    logger({ cnl: ["server"], act: "抓拍照片的大小：" + captureBlob.size });
    if (captureBlob.size < 48 * 48 || captureBlob.size >= 2 * 1024 * 1024) {
      // 经查以前记录，不完整图片均为8192大小。此处设置小于10KB的图片为未抓拍成功
      // 检查百度统计的记录后，这里的图片大小可能小于8192，也可能是有效的数据，所以降低图片大小的要求为face++的要求
      logger({
        cnl: ["server"],
        act: "摄像头异常",
        dtl: "定时抓拍照片大小异常",
        ext: { blobSize: captureBlob.size },
      });
      throw new Error("定时抓拍照片大小异常");
    }
    const startTime = Date.now();
    const [captureFilePath, signIdentifier] = await uploadToServer(captureBlob);
    const endTime = Date.now();
    logger({
      cnl: ["server"],
      act: "定时抓拍上传",
      ext: { cost: endTime - startTime },
    });
    await faceCompare(captureFilePath, signIdentifier, examRecordDataId);
    logger({
      cnl: ["server"],
      act: "定时抓拍比对",
      dtl: "定时抓拍流程成功",
      ext: { cost: Date.now() - endTime, signIdentifier },
    });
  } catch (error) {
    if (!(error instanceof Error)) {
      logger({
        cnl: ["server"],
        act: "snapAsync",
        dtl: "not an Error",
        stk: error + "",
      });
      return;
    }
    logger({
      cnl: ["server"],
      act: "定时抓拍流程失败",
      ejn: JSON.stringify(error),
      stk: error.stack,
      possibleError: error,
    });
    emit("on-async-recognize-result", {
      hasError: true,
      fileName: "",
    });
  }
}

type CameraInfo = {
  detail: string;
  pid: string;
  vid: string;
  name: string;
};
async function faceCompare(
  captureFilePath: string,
  signIdentifier: string,
  examRecordDataId: number
) {
  try {
    let cameraInfos;
    let hasVirtualCamera = false;
    if (typeof window.nodeRequire != "undefined") {
      const fs: typeof import("fs") = window.nodeRequire("fs");
      if (fileExists("multiCamera.exe")) {
        try {
          await execLocal("multiCamera.exe");
          cameraInfos = fs.readFileSync("CameraInfo.txt", "utf-8");
          // cameraInfos =
          //   '[{"detail":"@device:pnp:?display#int3470#4&300121c4&0&uid13424#{65e8773d-8f56-11d0-a3b9-00a0c9223196}{9c5f415a-02cd-4e28-aeb7-811cb317dd64}","name":"HP Truevision 5MP Front","pid":"13424","vid":"3470"},{"detail":"@device:pnp:?display#int3470#4&300121c4&0&uid13424#{65e8773d-8f56-11d0-a3b9-00a0c9223196}{a6c1c503-01f1-4767-a229-00a0b223162f}","name":"HP Truevision 8MP Rear","pid":"13424","vid":"3470"},{"detail":"@device:pnp:?usb#vid_8086&pid_0a80&mi_04#6&28913c47&0&0004#{65e8773d-8f56-11d0-a3b9-00a0c9223196}global","name":"Intel(R) RealSense(TM) 3D Camera (R200) RGB","pid":"0a80","vid":"8086"},{"detail":"@device:pnp:?usb#vid_8086&pid_0a80&mi_00#6&28913c47&0&0000#{65e8773d-8f56-11d0-a3b9-00a0c9223196}global","name":"Intel(R) RealSense(TM) 3D Camera (R200) Left-Right","pid":"0a80","vid":"8086"},{"detail":"@device:pnp:?usb#vid_8086&pid_0a80&mi_02#6&28913c47&0&0002#{65e8773d-8f56-11d0-a3b9-00a0c9223196}global","name":"Intel(R) RealSense(TM) 3D Camera (R200) Depth","pid":"0a80","vid":"8086"}]';
          if (cameraInfos && cameraInfos.trim()) {
            cameraInfos = cameraInfos.trim();
            cameraInfos = cameraInfos.replace(/\r\n/g, "");
            cameraInfos = cameraInfos.replace(/\n/g, "");
            logger({
              cnl: ["server"],
              act: "multiCamera.exe",
              ext: { cameraInfos },
            });
          }
          if (cameraInfos.includes('""')) {
            hasVirtualCamera = true;
          }
          // multiCamera.exe 1.0.1
          if (cameraInfos.includes("cameraInfo")) {
            cameraInfos = JSON.stringify(JSON.parse(cameraInfos).cameraInfo);
          }
          if (cameraInfos.length >= 800) {
            logger({
              cnl: ["server"],
              act: "multiCamera.exe",
              stk: "虚拟摄像头-cameraInfos超长",
              ext: { cameraInfos },
            });
            let ary: CameraInfo[] = JSON.parse(cameraInfos);
            // 相同pid&vid仅保留一个
            const pidAndVidCollector: string[] = [];
            ary = ary.filter((c) => {
              const pv = c.pid + "|" + c.vid;
              const res = pidAndVidCollector.includes(pv);
              pidAndVidCollector.push(pv);
              return !res;
            });
            cameraInfos = JSON.stringify(ary);
            logger({
              cnl: ["server"],
              act: "multiCamera.exe",
              stk: "除重复pid&vid",
            });
            if (cameraInfos.length >= 800) {
              cameraInfos = JSON.stringify(
                (<CameraInfo[]>JSON.parse(cameraInfos)).map((v) => {
                  return {
                    pid: v.pid,
                    vid: v.pid,
                    detail: "omitted",
                    name: v.name,
                  };
                })
              );
              console.log("摄像头检测超长：", "去除detail");
              console.log(cameraInfos);
            }
            if (cameraInfos.length >= 800) {
              logger({
                cnl: ["server"],
                act: "multiCamera.exe",
                stk: "精简后还是超长",
                ext: { cameraInfos },
              });
            }
          }
        } catch (error) {
          logger({
            cnl: ["server"],
            act: "multiCamera.exe",
            stk: "虚拟摄像头-读取摄像头列表失败",
            possibleError: error,
          });
          // throw new Error("读取摄像头列表失败");
        }
      }
    }

    let body: any = {
      fileUrl: captureFilePath,
      signIdentifier,
      examRecordDataId,
    };

    if (cameraInfos) {
      body.cameraInfos = cameraInfos;
      body.hasVirtualCamera = hasVirtualCamera;
    }
    logger({
      cnl: ["server"],
      act: "抓拍照片详细日志",
      ext: {
        fileUrl: captureFilePath,
        signIdentifier,
        examRecordDataId,
        cameraInfos,
        hasVirtualCamera,
      },
    });
    const res = await httpApp.post(
      "/api/ecs_oe_student_face/examCaptureQueue/uploadExamCapture",
      body
    );

    emit("on-async-recognize-result", {
      hasError: false,
      fileName: res.data,
    });
  } catch (e) {
    logger({
      cnl: ["server"],
      act: "定时抓拍",
      dtl: "抓拍失败",
      possibleError: e,
    });
    emit("on-async-recognize-result", {
      hasError: true,
      fileName: "",
    });

    throw new Error("异步比较抓拍照片失败");
  }
}

//#endregion 异步人脸比对
</script>

<template>
  <div>
    <video id="video" ref="video" :width="width" :height="height" autoplay />
    <div v-if="showRecognizeButton" class="btn-container">
      <button
        class="verify-button"
        :class="[snapBtnDisabled && 'disable-verify-button']"
        :disabled="snapBtnDisabled"
        @click="snapSync"
      >
        {{ btnText }}
      </button>
    </div>
  </div>
</template>

<style scoped>
.btn-container {
  position: absolute;
  width: 400px;
  text-align: center;
  margin-top: -50px;
  color: #232323;
}
.verify-button {
  font-size: 16px;
  background-color: #ffcc00;
  display: inline-block;
  padding: 6px 16px;
  border-radius: 6px;
}

.verify-button:hover {
  color: #444444;
  cursor: pointer;
}

.disable-verify-button {
  background-color: #f7f7f7;
  color: #c5c8ce;
}

.disable-verify-button:hover {
  cursor: not-allowed;
  color: #c5c8ce;
}
</style>