examcloud-web-photo-upload/src/views/index.vue

<template>
  <div style='height: 100%;'>
    <div class='import-header'>
      <img src='./images/nav_images.png' alt=''>照片批量上传工具 - {{server}}
      <button class='exit-btn' @click='logout' title="退出"></button>
    </div>
    <div class='import-body'>
      <div class='import-div'>
        <a href='javascript:;'>
          <!-- webkitdirectory-->
          <button class='import-btn' @click="importPhotos">
            上传照片
          </button>
        </a>
        <div>
          <input class="base-id-input" type="text" v-model="baseID" placeholder="不处理小于此数字的身份证照片">
        </div>
        <div class='progress-div'>
          <div>
            <span>
              成功(含跳过):
              <span style="color:green;">{{successNum}}({{skipNum}})</span>/{{allNum}}
            </span>
            <span>
              失败:
              <span style="color:red;">{{errorNum}}</span>/{{allNum}}
            </span>
            <span>
              并发参数:
              <span style="color:black;">{{batchConcurrency}}</span>
            </span>
            <span>
              并发请求:
              <span style="color:black;">{{reqNum}}</span>
            </span>
            <span>
              并发报警:
              <span style="color:red;">{{faceppConcurrencyErrorNum}}</span>
            </span>
            <span>
              报警频率:
              <span style="color:red;">{{faceppConcurrencyErrorNumPerMinute}}个/分</span>
            </span>
          </div>
          <div>
            <span>开始时间：{{startProcessTimeFormat}} </span>
            <span>结束时间：{{endProcessTimeFormat}} </span>
            <span>处理速度：{{(1000*processSpeed).toFixed(2)}}个/秒</span>
          </div>
          <div>
            <span v-show="completeShow" style="color: green;font-weight: bold;">全部处理完成</span>
            <span v-show="!completeShow&&(successNum>0||errorNum>0)" style="color: red;font-weight: bold;">处理中...</span>
          </div>
        </div>
      </div>
      <div class='console-panel'>
        <div v-for='msgInfo in returnMsgList200' :key="msgInfo.fileName">
          <p class='console-line'>
            <span>{{msgInfo.fileName}}:{{msgInfo.msg}}</span>
          </p>
        </div>
      </div>
    </div>
  </div>
</template>

<script>
const CryptoJS = require("crypto-js");
const Base64 = require("js-base64").Base64;
const fs = window.nodeRequire("fs");
const path = window.nodeRequire("path");
const async = require("async");
const moment = require("moment");

function isImageFile(fileName) {
  const lowercaseName = fileName.toLowerCase();
  return (
    lowercaseName.endsWith(".jpg") ||
    lowercaseName.endsWith(".jpeg") ||
    lowercaseName.endsWith(".png")
  );
}

// (async () => window.DB.updateFaceSet("fbd41e3e320f21c9f35467c268f86b4c", 22))();

//读取文件，返回网站路径. 只扫描当前文件夹，不递归扫描
function readImageFiles(folderPath) {
  const files = fs.readdirSync(folderPath);
  return files
    .filter(isImageFile)
    .sort()
    .map(fileName => folderPath + "/" + fileName);
}

let CONCURRENCY = 4; //同时处理的照片数量; 深夜增加并发

export default {
  data() {
    return {
      server: process.env.VUE_APP_ENV,
      photoList: [], //总的照片队列
      returnMsgList: [], //返回信息
      successNum: 0, //成功数量
      errorNum: 0, //失败数量
      skipNum: 0, //跳过数量
      allNum: 0, //总数
      completeShow: false,
      baseID: "", // 从大于这个身份证号开始处理，不能在init初始化
      reqNum: 0, // 并发请求数
      faceppConcurrencyErrorNum: 0, // face++的并发数报错
      startProcessTime: null, // 开始处理的时间
      endProcessTime: null, // 结束处理的时间
      processSpeed: 0, // 处理的速度：个/秒
      batchConcurrency: 0,
      faceppConcurrencyErrorNumPerMinute: 0 // 报警频率
    };
  },
  methods: {
    init() {
      this.photoList = [];
      this.returnMsgList = [];
      this.successNum = 0;
      this.errorNum = 0;
      this.skipNum = 0;
      this.allNum = 0;
      this.completeShow = false;
      this.reqNum = 0;
      this.faceppConcurrencyErrorNum = 0;
      this.startProcessTime = null;
      this.endProcessTime = null;
      this.processSpeed = 0;
      window.faceppConcurrencyErrorNum = 0;
    },
    importPhotos() {
      this.init();
      var electron = window.nodeRequire("electron");
      let dialog = electron.remote.dialog;
      dialog.showOpenDialog(
        {
          title: "选择照片所在目录",
          properties: ["openDirectory"]
        },
        folderPaths => {
          if (folderPaths) {
            this.photoList = readImageFiles(folderPaths[0]);
            this.allNum = this.photoList.length;
            this.startProcessTime = Date.now();
            this.processQueue();
          }
        }
      );
    },
    processQueue() {
      let taskQueue = [];
      let total = 0;
      for (
        let i = this.successNum + this.errorNum;
        i < this.photoList.length && total < 100;
        i++, total++
      ) {
        taskQueue.push(this.processStudentPhoto.bind(this, this.photoList[i]));
      }
      this.processQueueSingle(taskQueue);
    },
    processQueueSingle(taskQueue100) {
      // 并发处理请求
      // CONCURRENCY = new Date().getHours() < 6 ? 9 : 5;
      // 根据并发错误频率来决定下一次多少并发
      if (window.faceppConcurrencyErrorNumPerMinute < 7 && CONCURRENCY < 9) {
        CONCURRENCY++;
      } else {
        CONCURRENCY--;
      }

      this.batchConcurrency = CONCURRENCY;

      async.parallelLimit(taskQueue100, CONCURRENCY, (err, results) => {
        if (err) {
          alert(err);
          console.log(err);
        } else {
          console.log(results);
          if (this.successNum + this.errorNum < this.allNum) {
            console.log("处理完100张图片了，0.5秒后继续...间隔提供给GC");
            const delay = Date.now() - this.startProcessTime < 10000 ? 0 : 500; // 如果有跳过的图标则不等待0.5秒
            setTimeout(this.processQueue, delay);
          } else {
            console.log("photoList处理完毕");
            this.endProcessTime = Date.now();
            this.completeShow = true;
          }
        }
      });
    },
    //处理单个照片
    async processStudentPhoto(studentPhotoPath) {
      const fileSuffix = path.extname(studentPhotoPath); //文件后缀
      const identityNumber = path
        .basename(studentPhotoPath)
        .replace(fileSuffix, ""); //文件名就是身份证号码

      // 根据用户输入来跳过部分图片
      if (this.lessThanBaseID(identityNumber)) {
        this.finishOnePhotoSuccess("跳过处理", studentPhotoPath);
        this.skipNum++;
        return;
      }
      const photoFile = fs.readFileSync(studentPhotoPath);
      const rootOrgId = localStorage.getItem("rootOrgId");

      //生成新名称
      let serverPhotoPath = null;
      const upyunPhotoPath = (() => {
        const md5Hash = CryptoJS.MD5(
          Base64.encode(identityNumber + new Date().getTime())
        ).toString();
        serverPhotoPath = md5Hash + fileSuffix;
        return (
          rootOrgId +
          "/" +
          encodeURIComponent(identityNumber) +
          "/" +
          md5Hash +
          fileSuffix
        );
      })();

      // 核心流程：
      // 1. get studentId from ecs
      // 2. get faceToken from facepp
      // 3. get faceSetToken from ecs
      // 4. add faceToken to faceSetToken
      // 5. save photo to upyun
      // 6. 根据以上信息，保存到服务器
      // 每一步出错都会保存到错误日志
      try {
        // 不用Promise.all的原因是每一步失败就不用进行下一步了
        let studentId = await this.getStudentId(rootOrgId, identityNumber);
        let faceToken = await this.detectFace(photoFile);
        this.faceSetToken = await this.getFaceSetToken();
        let faceCount = await this.addFaceToSet(this.faceSetToken, faceToken);

        await this.saveImageToUpyun({ upyunPhotoPath, photoFile });
        const photoInfo = {
          studentId: studentId,
          faceSetToken: this.faceSetToken,
          faceToken: faceToken,
          studentPhotoPath: studentPhotoPath,
          rootOrgId: rootOrgId,
          faceCount,
          photoName: serverPhotoPath
        };
        await this.saveStudentFaceInfoByPut(photoInfo);
        this.finishOnePhotoSuccess("处理成功", studentPhotoPath);

        {
          // 执行过程中的元信息
          this.reqNum = window.requestInProcessingTotal;
          this.faceppConcurrencyErrorNum = window.faceppConcurrencyErrorNum;
          this.faceppConcurrencyErrorNumPerMinute =
            window.faceppConcurrencyErrorNumPerMinute;
          this.processSpeed =
            (this.successNum + this.errorNum - this.skipNum) /
            (Date.now() - this.startProcessTime);
        }
      } catch (err) {
        console.log(err);
        this.finishOnePhotoFail(err, studentPhotoPath);
      }
    },
    async getStudentId(rootOrgId, identityNumber) {
      return new Promise((resolve, reject) => {
        this.$http
          .get(
            "/api/ecs_core/student/getStudentInfo?orgId=" +
              rootOrgId +
              "&identityNumber=" +
              identityNumber
          )
          .then(res => {
            var studentFaceInfo = res.data;
            if (studentFaceInfo && studentFaceInfo.id) {
              resolve(studentFaceInfo.id);
            } else {
              reject("查询身份证不存在");
            }
          })
          .catch(err => {
            console.log(err);
            reject("根据身份证号码查询失败");
          });
      });
    },
    //保存文件至又拍云
    async saveImageToUpyun({ upyunPhotoPath, photoFile }) {
      const url = process.env.VUE_APP_UPYUN_BUCKETURL + upyunPhotoPath;
      const authorization =
        "Basic " +
        Base64.encode(
          process.env.VUE_APP_UPYUN_OPERATOR +
            ":" +
            process.env.VUE_APP_UPYUN_PASSWORD
        );
      const headers = {
        headers: {
          Authorization: authorization,
          "Content-Type": "image/jpeg"
        }
      };
      return this.$http.put(url, photoFile, headers).catch(err => {
        console.log(err);
        throw "saveImageToUpyun失败";
      });
    },
    //获取faceSetToken
    async getFaceSetToken() {
      if (this.faceSetToken) {
        return this.faceSetToken;
      } else {
        return new Promise((resolve, reject) => {
          this.$http
            .get("/api/ecs_core/face/getUsableFacesetList")
            .then(res => {
              if (res.data.length < 1) {
                reject("获取facesetToken失败: 没有可用的facesetToken");
              } else {
                resolve(res.data[0].facesetToken);
              }
            })
            .catch(err => {
              console.log(err);
              reject("获取facesetToken失败: 接口出错");
            });
        });
      }
    },
    //faceToken加入faceSetToken
    async addFaceToSet(faceset_token, face_token) {
      let formData_addface = new FormData();
      formData_addface.append("api_key", process.env.VUE_APP_FACEPP_API_KEY);
      formData_addface.append(
        "api_secret",
        process.env.VUE_APP_FACEPP_API_SECRET
      );
      formData_addface.append("faceset_token", faceset_token);
      formData_addface.append("face_tokens", face_token);
      return new Promise((resolve, reject) => {
        this.$http
          .post("/facepp/v3/faceset/addface", formData_addface)
          .then(res => {
            // console.log(
            //   `res.data.face_added: ${
            //     res.data.face_added
            //   }, res.data.face_count: ${res.data.face_count}`
            // );
            if (res.data.face_added !== 1) {
              reject(
                "faceToken加入faceSetToken失败: face_added为" +
                  res.data.face_added
              );
            }
            if (res.data.face_count > 8000) {
              this.faceSetToken = undefined;
            }
            resolve(res.data.face_added);
          })
          .catch(err => {
            console.log(err);
            reject("faceToken加入faceSetToken失败: addface catch error");
          });
      });
    },
    //face++分析人脸
    async detectFace(file) {
      let fileBlob = new Blob([file]);
      let formData_face_token = new FormData();
      formData_face_token.append("api_key", process.env.VUE_APP_FACEPP_API_KEY);
      formData_face_token.append(
        "api_secret",
        process.env.VUE_APP_FACEPP_API_SECRET
      );
      formData_face_token.append("image_file", fileBlob);
      return new Promise((resolve, reject) => {
        this.$http
          .post("/facepp/v3/detect", formData_face_token)
          .then(res => {
            if (res.data.faces.length >= 1) {
              resolve(res.data.faces[0].face_token);
            } else {
              reject("face++没有检测到人脸;");
            }
          })
          .catch(err => {
            console.log(err);
            reject("调用face++检测人脸失败");
          });
      });
    },
    async saveStudentFaceInfoByPut({
      rootOrgId,
      studentId,
      faceSetToken,
      faceToken,
      faceCount,
      photoName
    }) {
      return this.$http
        .post("/api/ecs_core/face/saveStudentFace", {
          rootOrgId,
          studentId,
          facesetToken: faceSetToken,
          faceToken,
          faceCount,
          photoName,
          operator: "客户端工具上传-" + localStorage.getItem("userName")
        })
        .catch(err => {
          console.log(err);
          throw "saveStudentFaceInfoByPut失败";
        });
    },
    //成功或失败处理
    finishOnePhotoFail(msg, studentPhotoPath) {
      try {
        const fileName = path.basename(studentPhotoPath);
        this.returnMsgList.push({
          success: false,
          fileName,
          msg
        });
        this.errorNum++;

        //移动照片到errorfiles文件夹
        const errorfilePath = path.join(
          path.dirname(studentPhotoPath),
          "errorfiles"
        );
        if (!fs.existsSync(errorfilePath)) {
          fs.mkdirSync(errorfilePath);
        }
        fs.copyFileSync(studentPhotoPath, path.join(errorfilePath, fileName));

        fs.appendFileSync(
          path.join(errorfilePath, "errorPhotos.txt"),
          fileName + ":" + msg + "\n"
        );
      } catch (error) {
        console.log(error);
      }
    },
    //成功处理一张照片
    finishOnePhotoSuccess(msg, studentPhotoPath) {
      this.returnMsgList.push({
        success: true,
        fileName: path.basename(studentPhotoPath),
        msg: msg
      });
      this.successNum++;
      fs.appendFileSync(
        path.join(path.dirname(studentPhotoPath), "successPhotos.txt"),
        path.basename(studentPhotoPath) + ":" + msg + "\n"
      );
    },
    logout() {
      localStorage.removeItem("rootOrgId");
      localStorage.removeItem("userName");
      localStorage.removeItem("user_token");
      this.$router.push({
        path: "/login"
      });
    },
    lessThanBaseID(identityNumber) {
      return identityNumber < this.baseID; //字符串比较，从第一个字符比较起
    }
  },
  computed: {
    returnMsgList200() {
      return this.returnMsgList.slice(this.returnMsgList.length - 200);
    },
    startProcessTimeFormat() {
      return this.startProcessTime
        ? moment(this.startProcessTime).format("YYYY-MM-DD HH:mm:ss")
        : "-";
    },
    endProcessTimeFormat() {
      return this.endProcessTime
        ? moment(this.endProcessTime).format("YYYY-MM-DD HH:mm:ss")
        : "-";
    }
  }
};
</script>

<style scoped>
.import-header {
  background-color: #3ed798;
  color: white;
  height: 80px;
  padding: 0 30px;
  line-height: 80px;
  text-align: left;
}

.import-header > img {
  height: 34px;
  margin-right: 10px;
}

.exit-btn {
  position: absolute;
  top: 0;
  right: 0;
  border: none;
  width: 60px;
  height: 36px;
  background: url("./images/btn_closed.png") no-repeat center;
  background-size: cover;
  outline: none;
}

.exit-btn:hover {
  background: url("./images/btn_closed_hover.png") no-repeat center;
  cursor: pointer;
}

.import-body {
  display: grid;
  height: calc(100vh - 100px);
  grid-template-rows: 270px 1fr;
}

.import-div {
  text-align: center;
  padding: 20px 0;
}

.import-btn {
  width: 316px;
  height: 70px;
  border-radius: 35px;
  border: none;
  font-size: 30px;
  color: #e3e3e3;
  background-color: #3ed798;
}

.import-btn:hover {
  color: #777777;
  cursor: pointer;
}

.base-id-input {
  margin-top: 10px;
  width: 300px;
  font-size: 20px;
}

.progress-div {
  display: flex;
  flex-direction: column;
  line-height: 40px;
  margin: 20px 0px;
  font-size: 20px;
  border: 1px solid gainsboro;
  cursor: default;
}

.console-panel {
  background-color: #e3e3e3;
  overflow: auto;
}

.console-panel .console-line {
  font-size: 14px;
  text-align: center;
  display: flex;
  justify-content: center;
  align-items: center;
}

.console-panel .console-line > span {
  margin: 0 10px;
}

.console-panel .console-line > span.red {
  color: #fc7156;
}

.console-panel .console-line > span.green {
  color: #3ed798;
}
</style>