新增图片检查功能

config.json

@@ -8,6 +8,7 @@
     },
     "imageUrl": {
         "sheet": "/{{examId}}-{{campusCode}}/{{subjectCode}}/{{examNumber}}-{{index}}.jpg",
+        "slice": "/{{examId}}-{{campusCode}}/{{subjectCode}}/{{examNumber}}-{{index}}.jpg",
         "package": "/{{examId}}/{{code}}/{{index}}.jpg"
     },
     "watermark": {

source/lib/api.js

@@ -14,7 +14,7 @@ async function execute(uri, method, form) {
                 'auth-info': 'loginname=' + env.loginName + ';password=' + env.password
             },
             maxAttempts: 50,
-            retryDelay: 1000,
+            retryDelay: 500,
             retryStrategy: request.RetryStrategies.HTTPOrNetworkError
         }, function(error, response, body) {
             if (response.statusCode == 200) {

source/lib/image.js

@@ -6,12 +6,14 @@ const config = require('./config.js')
 const logger = require('./logger.js')('image.js')
 const downloadLogger = require('./logger.js')('download')
 const upyun = require('./upyun.js')
+const PromisePool = require('./promise-pool.js')
 const fs = require('fs')
 const path = require('path')
 const readline = require('readline')
 const sizeOf = require('image-size')
 const mustache = require('mustache')
 const mkdirp = require('mkdirp')
+const moment = require('moment')
 const gm = config.imagemagick != undefined ? require('gm').subClass({
     imageMagick: true,
     appPath: config.imagemagick
@@ -38,6 +40,13 @@ class executor extends EventEmitter {
         })
     }
 
+    async checkFile(url, client) {
+        let size = sizeOf(await client.download(url))
+        if (size.width == 0 || size.height == 0) {
+            throw 'invalid image data:' + url
+        }
+    }
+
     async addWatermark(image, file, student, index, showMarker) {
         let fontFile = config.watermark.fontFile
         let color = config.watermark.color
@@ -282,6 +291,122 @@ class executor extends EventEmitter {
             this.emit('error', error)
         }
     }
+
+    async checkSlice(dir, concurrent) {
+        let bucket = env.server.bucketPrefix + '-slice'
+        let client = upyun(bucket, config.upyun.operator, config.upyun.password)
+        if (env.server.upyunDomain && env.server.upyunDomain != '') {
+            //局域网模式修改图片服务器地址
+            client.setDomain(env.server.upyunDomain)
+        }
+
+        try {
+            let logFile = path.join(dir, 'result.txt')
+            fs.writeFileSync(logFile, moment().format('YYYY-MM-DD HH:mm:ss') + ', examId=' + env.examId + '\r\n')
+
+            let totalCount = await api.countStudents(env.examId, true, false)
+            this.emit('total', totalCount)
+
+            let self = this
+            let count = 0
+            let pageNumber = 0
+            let pool = PromisePool.create(concurrent, function(student) {
+                student.examId = env.examId
+                student.promises = []
+                for (let i = 1; i <= student.sliceCount; i++) {
+                    student.index = i
+                    let url = mustache.render(config.imageUrl.slice, student)
+                    student.promises.push(new Promise(resolved => {
+                        self.checkFile(url, client).then(() => {
+                            //fs.appendFileSync(logFile, url + ': success\r\n')
+                        }).catch(() => {
+                            fs.appendFileSync(logFile, url + ': error\r\n')
+                        }).finally(() => {
+                            resolved()
+                        })
+                    }))
+                }
+                return Promise.all(student.promises)
+            })
+            pool.on('count', function(offset) {
+                self.emit('count', count + offset)
+            })
+
+            this.emit('count', 0)
+            for (;;) {
+                pageNumber++
+                let array = await api.getStudents(env.examId, pageNumber, 200, true, false, false, false)
+                if (array == undefined || array.length == 0) {
+                    break
+                }
+                await pool.start(array)
+                count += array.length
+            }
+            this.emit('finish')
+            fs.appendFileSync(logFile, moment().format('YYYY-MM-DD HH:mm:ss') + ', examId=' + env.examId)
+        } catch (error) {
+            logger.error('check slice error:' + error)
+            logger.error(error)
+            this.emit('error', error)
+        }
+    }
+
+    async checkSliceSerial(dir) {
+        let bucket = env.server.bucketPrefix + '-slice'
+        let client = upyun(bucket, config.upyun.operator, config.upyun.password)
+        if (env.server.upyunDomain && env.server.upyunDomain != '') {
+            //局域网模式修改图片服务器地址
+            client.setDomain(env.server.upyunDomain)
+        }
+
+        try {
+            let logFile = path.join(dir, 'result.txt')
+            fs.writeFileSync(logFile, moment().format('YYYY-MM-DD HH:mm:ss') + ', examId=' + env.examId + '\r\n')
+
+            let totalCount = await api.countStudents(env.examId, true, false)
+            this.emit('total', totalCount)
+
+            let self = this
+            let count = 0
+            let pageNumber = 0
+            this.emit('count', 0)
+            for (;;) {
+                pageNumber++
+
+                let array = await api.getStudents(env.examId, pageNumber, 100, true, false, false, false)
+                if (array == undefined || array.length == 0) {
+                    break
+                }
+                for (let i = 0; i < array.length; i++) {
+                    let student = array[i]
+                    student.examId = env.examId
+                    student.promises = []
+                    for (let i = 1; i <= student.sliceCount; i++) {
+                        student.index = i
+                        let url = mustache.render(config.imageUrl.slice, student)
+                        student.promises.push(new Promise(resolved => {
+                            self.checkFile(url, client).then(() => {
+                                //fs.appendFileSync(logFile, url + ': success\r\n')
+                            }).catch(() => {
+                                fs.appendFileSync(logFile, url + ': error\r\n')
+                            }).finally(() => {
+                                resolved()
+                            })
+                        }))
+                    }
+                    await Promise.all(student.promises)
+                    count++
+                    this.emit('count', count)
+                }
+            }
+            this.emit('finish')
+            fs.appendFileSync(logFile, moment().format('YYYY-MM-DD HH:mm:ss') + ', examId=' + env.examId)
+        } catch (error) {
+            logger.error('check slice error:' + error)
+            logger.error(error)
+            this.emit('error', error)
+        }
+    }
 }
 
 module.exports = function() {

source/lib/promise-pool.js

@@ -0,0 +1,39 @@
+//Promise并发控制池
+const EventEmitter = require('events')
+
+class PromisePool extends EventEmitter {
+
+    constructor(concurrent, promiseCreator) {
+        super()
+        this.concurrent = concurrent || 5
+        this.promiseCreator = promiseCreator
+    }
+
+    start(datas) {
+        let self = this
+        let index = 0
+        let running = 0
+        return new Promise(resolved => {
+            let execute = () => {
+                running++
+                self.promiseCreator(datas[index++]).finally(() => {
+                    self.emit('count', index)
+                    running--
+                    if (running < self.concurrent && datas.length > index) {
+                        execute()
+                    } else if (datas.length == index && running == 0) {
+                        resolved()
+                    }
+                })
+            }
+            while (running < self.concurrent && datas.length > index) {
+                execute()
+            }
+        })
+    }
+
+}
+
+exports.create = function(concurrent, promiseCreator) {
+    return new PromisePool(concurrent, promiseCreator)
+}

source/lib/upyun.js

@@ -346,7 +346,7 @@ module.exports = util.Class(function() {
                 timeout: 120000,
                 headers: headers,
                 maxAttempts: 5,
-                retryDelay: 1000,
+                retryDelay: 500,
                 retryStrategy: request.RetryStrategies.HTTPOrNetworkError
             }, options);
             request(opts, function(error, response, body) {

source/test/sizeof-test.js

@@ -0,0 +1,3 @@
+const sizeOf = require('image-size')
+
+console.log(sizeOf('/Users/luoshi/Downloads/19082981-1.jpg'))

source/view/check.html

@@ -0,0 +1,104 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8">
+    <title>云阅卷本地代理工具</title>
+    <meta name="viewport" content="width=device-width,initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0, user-scalable=no" />
+    <link rel="stylesheet" href="css/style.css">
+</head>
+
+<body>
+    <div class="wp">
+        <div class="hd">
+            <div class="logo"><img src="img/logo.png" /></div>
+            <span class="y"> 欢迎您，<span id="user-name"></span>
+                <span id="school-switch" style="display:none">
+                    <span class="pipe">|</span><a href="school-list.html"></a>
+                </span>
+                <span class="pipe">|</span><a href="login.html">退出</a>
+            </span>
+        </div>
+        <div class="cont">
+            <div class="title cl">
+                <span class="y"><a href="index.html">返回考试主页</a></span>
+                <h2>图片检查</h2>
+            </div>
+            <div class="picture cl">
+                <table cellpadding="0" cellspacing="0" width="100%">
+                    <tr>
+                        <th>结果保存目录：</th>
+                        <td>
+                            <input id="path-text" type="text" style="width: 400px" class="filetext" />
+                            <a href="##" id="path-select" class="filebtn">选择</a>
+                        </td>
+                    </tr>
+                    <tr>
+                        <th>并发数：</th>
+                        <td><input id="concurrent-input" type="text" style="width: 100px" value="1" /></td>
+                    </tr>
+                    <tr id="message-tr" style="display: none">
+                        <th></th>
+                        <td>
+                            <p class="error-tetx" id="message-text"></p>
+                        </td>
+                    </tr>
+                    <tr>
+                        <th></th>
+                        <td><a id="run-button" href="##" class="start-btn"><span>开始检查</span></a></td>
+                    </tr>
+                </table>
+            </div>
+        </div>
+        <div class="ft">Copyright © 2011-2020 www.qmth.com.cn, All Rights Reserved</div>
+    </div>
+
+    <script>
+        const $ = require('jquery')
+        const env = require('../lib/env.js')
+        const config = require('../lib/config.js')
+        const {
+            dialog
+        } = require('electron').remote
+
+        $(document).ready(() => {
+            env.merge(JSON.parse(window.localStorage.getItem('env')))
+            $('#user-name').html(env.user.userName)
+
+            let schoolName = env.getSchoolName()
+            if (schoolName != undefined) {
+                $('#school-switch').find('a').html(schoolName)
+                $('#school-switch').show()
+            }
+
+            $('#path-select').click(() => {
+                dialog.showOpenDialog({
+                    title: '请选择保存目录',
+                    properties: ['openDirectory']
+                }, filePaths => {
+                    $('#path-text').val(filePaths[0])
+                })
+            })
+
+            $('#run-button').click(() => {
+                let dir = $('#path-text').val()
+                let concurrent = $('#concurrent-input').val()
+                if (dir == undefined || dir == '') {
+                    alert('请选择保存目录')
+                    return false
+                }
+                if (concurrent == undefined || concurrent == '') {
+                    alert('请填写并发数')
+                    return false
+                }
+                window.localStorage.setItem('check-config', JSON.stringify({
+                    dir: dir.trim(),
+                    concurrent: parseInt(concurrent)
+                }))
+                window.location.href = 'image-check.html'
+            })
+        })
+    </script>
+</body>
+
+</html>

source/view/image-check.html

@@ -0,0 +1,104 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8">
+    <title>云阅卷本地代理工具</title>
+    <meta name="viewport" content="width=device-width,initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0, user-scalable=no" />
+    <link rel="stylesheet" href="css/style.css">
+</head>
+
+<body>
+    <div class="wp">
+        <div class="hd">
+            <div class="logo"><img src="img/logo.png" /></div>
+            <span class="y"> 欢迎您，<span id="user-name"></span>
+                <span id="school-switch" style="display:none">
+                    <span class="pipe">|</span><a href="##"></a>
+                </span>
+                <span class="pipe">|</span><a href="##">退出</a>
+            </span>
+        </div>
+        <div class="cont">
+            <div class="title title_grey cl">
+                <h2>图片检查中 …</h2>
+            </div>
+            <div class="progress-box">
+                <h3>正在检查图片，请耐心等候 ~</h3>
+                <div class="progress">
+                    <div class="progress-outer">
+                        <div id="progress" class="progress-inner" style="width: 0%;"><span class="progress-text"></span></div>
+                    </div>
+                </div>
+                <p>已检查人数：<b id="finish-count"></b> / 全部人数：<b id="total-count"></b></p>
+            </div>
+        </div>
+        <div class="xcConfirm" id="popup" style="display: none">
+            <div class="xc_layer"></div>
+            <div class="popbox">
+                <a href="##" id="popup-close"><span class="close"></span></a>
+                <div class="txtbox">
+                    <div id="popup-error" class="icon error" style="display: none"></div>
+                    <div id="popup-success" class="icon success" style="display: none"></div>
+                    <div id="popup-text" class="text"></div>
+                </div>
+            </div>
+        </div>
+        <div class="ft">Copyright © 2011-2020 www.qmth.com.cn, All Rights Reserved</div>
+    </div>
+
+    <script>
+        const $ = require('jquery')
+        const env = require('../lib/env.js')
+        const imageUtil = require('../lib/image.js')()
+
+        $(document).ready(() => {
+            env.merge(JSON.parse(window.localStorage.getItem('env')))
+            $('#user-name').html(env.user.userName)
+
+            let schoolName = env.getSchoolName()
+            if (schoolName != undefined) {
+                $('#school-switch').find('a').html(schoolName)
+                $('#school-switch').show()
+            }
+
+            let config = JSON.parse(window.localStorage.getItem('check-config'))
+            let totalCount;
+            imageUtil.on('total', (count) => {
+                totalCount = count
+                $('#total-count').html(count)
+            })
+            imageUtil.on('count', (count) => {
+                $('#finish-count').html(count)
+                let rate = parseInt(count * 100 / totalCount)
+                $('#progress').css('width', rate + '%')
+                $('.progress-text').html(rate + '%')
+            })
+            imageUtil.on('finish', () => {
+                $('#popup-success').show()
+                $('#popup-text').html('图片检查完成')
+                $('#popup-close').click(() => {
+                    $('#popup').hide()
+                    window.location.href = 'check.html'
+                })
+                $('#popup').show()
+            })
+            imageUtil.on('error', (err) => {
+                $('#popup-error').show()
+                $('#popup-text').html('图片检查出错\n' + (err || ''))
+                $('#popup-close').click(() => {
+                    $('#popup').hide()
+                    window.location.href = 'check.html'
+                })
+                $('#popup').show()
+            })
+            if (config.concurrent == 1) {
+                imageUtil.checkSliceSerial(config.dir)
+            } else {
+                imageUtil.checkSlice(config.dir, config.concurrent)
+            }
+        })
+    </script>
+</body>
+
+</html>

source/view/index.html

@@ -30,6 +30,7 @@
                     <li class="l1"><a href="exam-list.html"><span></span>考试切换</a></li>
                     <li class="l2"><a href="sync.html"><span></span>数据同步</a></li>
                     <li class="l3"><a href="image.html"><span></span>图片下载</a></li>
+                    <li class="l2"><a href="check.html"><span></span>图片检查</a></li>
                 </ul>
             </div>
         </div>