大量功能优化完善

针对缺失图片单独记录日志并跳过，不跳出整个下载循环；
request库修改为requestretry库，增加http请求超时重试处理；
从又拍云下载图片改为https连接

config.json

@@ -20,11 +20,7 @@
     },
     "openDevTools": false,
     "localStore": "",
-    "syncTime": {
-        "192.168.10.42:8080_1": "2018-10-19 18:40:47"
-    },
-    "servers": [
-        {
+    "servers": [{
             "name": "高校",
             "host": "gx.markingcloud.com",
             "bucketPrefix": "gx",
@@ -43,4 +39,4 @@
             "upyunDomain": "192.168.10.42:8080/file"
         }
     ]
-}
+}

source/lib/api.js

@@ -1,4 +1,4 @@
-const request = require('request')
+const request = require('requestretry')
 const env = require('./env.js')
 const logger = require('./logger.js')('api.js')
 
@@ -8,10 +8,14 @@ async function execute(uri, method, form) {
             url: 'http://' + env.server.host + uri,
             method: method,
             json: true,
+            timeout: 10000,
             form: form || {},
             headers: {
                 'auth-info': 'loginname=' + env.loginName + ';password=' + env.password
-            }
+            },
+            maxAttempts: 50,
+            retryDelay: 1000,
+            retryStrategy: request.RetryStrategies.HTTPOrNetworkError
         }, function (error, response, body) {
             if (response.statusCode == 200) {
                 resolve(body);
@@ -82,4 +86,4 @@ module.exports.getPackages = function (examId, upload) {
         uri = uri + '?upload=' + (upload ? 'true' : false)
     }
     return execute(uri, 'GET');
-}
+}

source/lib/image.js

@@ -4,6 +4,7 @@ const api = require('./api.js')
 const env = require('./env.js')
 const config = require('./config.js')
 const logger = require('./logger.js')('image.js')
+const downloadLogger = require('./logger.js')('download')
 const upyun = require('./upyun.js')
 const fs = require('fs')
 const path = require('path')
@@ -114,11 +115,21 @@ class executor extends EventEmitter {
             let cache = path.join(config.localStore, bucket, remote)
             if (fs.existsSync(cache)) {
                 imgData = fs.readFileSync(cache)
-            } else {
+            }
+        }
+        if (imgData == undefined) {
+            try {
                 imgData = await client.download(remote)
+            } catch (err) {
+                if (err.code === 404) {
+                    //文件不存在，记录日志并跳过
+                    downloadLogger.error('404 ' + bucket + ' ' + remote)
+                    return Promise.resolve()
+                } else {
+                    logger.error(err)
+                    return Promise.reject(err)
+                }
             }
-        } else {
-            imgData = await client.download(remote)
         }
         //是否需要添加分数水印
         if (watermark) {
@@ -132,7 +143,7 @@ class executor extends EventEmitter {
         //从文件中读取已成功下载的准考证号列表
         let doneFile = path.join(__dirname, '../../logs/sheet.log')
         let finish = await this.readFile(doneFile)
-        logger.info(finish)
+        //logger.info(finish)
         let success = {}
         for (let i = 0; i < finish.length; i++) {
             success[finish[i]] = true
@@ -174,23 +185,26 @@ class executor extends EventEmitter {
                         //等待所有图片下载完毕
                         await Promise.all(promises)
 
-                        finish.push(examNumber)
+                        //finish.push(examNumber)
                         success[examNumber] = true
+
+                        //实时回写到记录文件中去
+                        fs.appendFileSync(doneFile, examNumber + '\r\n')
                     }
                     count++
                     this.emit('count', count)
 
                     //实时回写到记录文件中去
-                    fs.writeFileSync(doneFile, finish.join('\r\n'))
+                    //fs.writeFileSync(doneFile, finish.join('\r\n'))
                 }
             }
             this.emit('finish')
         } catch (error) {
-            logger.error(error)
+            logger.error('download sheet error:' + error)
             this.emit('error', error)
         } finally {
             //回写到记录文件中去
-            fs.writeFileSync(doneFile, finish.join('\r\n'))
+            //fs.writeFileSync(doneFile, finish.join('\r\n'))
         }
     }
 
@@ -218,7 +232,7 @@ class executor extends EventEmitter {
             }
             this.emit('finish')
         } catch (err) {
-            logger.error(err)
+            logger.error('download package error: ' + err)
             this.emit('error', err)
         }
     }

source/lib/logger.js

@@ -7,12 +7,20 @@ log4js.configure({
         everything: {
             type: 'dateFile',
             filename: path.join(__dirname, '../../logs/app.log')
+        },
+        download: {
+            type: 'file',
+            filename: path.join(__dirname, '../../logs/download.log')
         }
     },
     categories: {
         default: {
             appenders: ['everything'],
             level: config.logger.level
+        },
+        download: {
+            appenders: ['download'],
+            level: config.logger.level
         }
     }
 })

source/lib/upyun.js

@@ -1,52 +1,58 @@
 var util = require("thinkjs-util");
-var request = require("request");
+var request = require("requestretry");
 var fs = require("fs");
 var path = require("path");
 
 module.exports = util.Class(function () {
     return {
-		/**
-		 * 接口的域名
-		 * @type {String}
-		 */
-        domain: "v0.api.upyun.com",
-		/**
-		 * 初始化
-		 * @param  {[type]} bucketname [description]
-		 * @param  {[type]} username   [description]
-		 * @param  {[type]} password   [description]
-		 * @return {[type]}            [description]
-		 */
+        /**
+         * 默认接口域名
+         * @type {String}
+         */
+        DEFAULT_DOMAIN: "v0.api.upyun.com",
+        /**
+         * 接口的域名
+         * @type {String}
+         */
+        domain: this.DEFAULT_DOMAIN,
+        /**
+         * 初始化
+         * @param  {[type]} bucketname [description]
+         * @param  {[type]} username   [description]
+         * @param  {[type]} password   [description]
+         * @return {[type]}            [description]
+         */
         init: function (bucketname, username, password) {
             this.bucketname = bucketname;
             this.username = username;
             this.password = util.md5(password);
+            this.domain = this.DEFAULT_DOMAIN;
         },
-		/**
-		 * 设置接口的domain
-		 * @param {[type]} domain [description]
-		 */
+        /**
+         * 设置接口的domain
+         * @param {[type]} domain [description]
+         */
         setDomain: function (domain) {
             this.domain = domain;
             return this;
         },
-		/**
-		 * 签名
-		 * @param  {[type]} method [description]
-		 * @param  {[type]} uri    [description]
-		 * @param  {[type]} date   [description]
-		 * @param  {[type]} length [description]
-		 * @return {[type]}        [description]
-		 */
+        /**
+         * 签名
+         * @param  {[type]} method [description]
+         * @param  {[type]} uri    [description]
+         * @param  {[type]} date   [description]
+         * @param  {[type]} length [description]
+         * @return {[type]}        [description]
+         */
         sign: function (method, uri, date, length) {
             var sign = method + '&' + uri + '&' + date + '&' + length + '&' + this.password;
             return 'UpYun ' + this.username + ':' + util.md5(sign);
         },
-		/**
-		 * 获取文件或者文件夹的信息
-		 * @param  {[type]} file [description]
-		 * @return {[type]}      [description]
-		 */
+        /**
+         * 获取文件或者文件夹的信息
+         * @param  {[type]} file [description]
+         * @return {[type]}      [description]
+         */
         getInfo: function (file) {
             return this.request(file, 'HEAD').then(function (response) {
                 var headers = response.headers;
@@ -57,21 +63,21 @@ module.exports = util.Class(function () {
                 };
             });
         },
-		/**
-		 * 查看空间占用信息
-		 * @return {[type]} [description]
-		 */
+        /**
+         * 查看空间占用信息
+         * @return {[type]} [description]
+         */
         getUsage: function (path) {
             path = path || "/";
             return this.request(path + "?usage").then(function (response) {
                 return parseInt(response.body, 10);
             });
         },
-		/**
-		 * 从返回的headers里获取图片的信息
-		 * @param  {[type]} response [description]
-		 * @return {[type]}          [description]
-		 */
+        /**
+         * 从返回的headers里获取图片的信息
+         * @param  {[type]} response [description]
+         * @return {[type]}          [description]
+         */
         getPicInfo: function (response) {
             var headers = response.headers;
             return {
@@ -81,12 +87,12 @@ module.exports = util.Class(function () {
                 type: headers['x-upyun-file-type']
             };
         },
-		/**
-		 * 上传文件或者文件夹
-		 * @param  {[type]} savePath [description]
-		 * @param  {[type]} filePath [description]
-		 * @return {[type]}          [description]
-		 */
+        /**
+         * 上传文件或者文件夹
+         * @param  {[type]} savePath [description]
+         * @param  {[type]} filePath [description]
+         * @return {[type]}          [description]
+         */
         upload: function (savePath, filePath, headers) {
             var defaultHeaders = {
                 mkdir: true
@@ -142,12 +148,12 @@ module.exports = util.Class(function () {
                 });
             }
         },
-		/**
-		 * 文件或者文件夹下载
-		 * @param  {[type]} path     [description]
-		 * @param  {[type]} savePath [description]
-		 * @return {[type]}          [description]
-		 */
+        /**
+         * 文件或者文件夹下载
+         * @param  {[type]} path     [description]
+         * @param  {[type]} savePath [description]
+         * @return {[type]}          [description]
+         */
         download: function (sourcePath, savePath, typeData) {
             sourcePath = sourcePath || "/";
             //if (savePath && savePath.slice(-1) !== "/") {
@@ -201,12 +207,12 @@ module.exports = util.Class(function () {
                 }
             });
         },
-		/**
-		 * 删除文件或者文件夹
-		 * @param  {[type]} path  [description]
-		 * @param  {[type]} force [description]
-		 * @return {[type]}       [description]
-		 */
+        /**
+         * 删除文件或者文件夹
+         * @param  {[type]} path  [description]
+         * @param  {[type]} force [description]
+         * @return {[type]}       [description]
+         */
         rm: function (path, force) {
             if (!path) {
                 return util.getPromise(new Error("path can't empty"), true);
@@ -248,11 +254,11 @@ module.exports = util.Class(function () {
                 }
             });
         },
-		/**
-		 * 递归创建目录
-		 * @param  {[type]} path [description]
-		 * @return {[type]}      [description]
-		 */
+        /**
+         * 递归创建目录
+         * @param  {[type]} path [description]
+         * @return {[type]}      [description]
+         */
         mkDir: function (path) {
             return this.request(path, 'PUT', '', {
                 mkdir: true,
@@ -261,11 +267,11 @@ module.exports = util.Class(function () {
                 return response.body;
             });
         },
-		/**
-		 * 读取目录下的文件和子目录
-		 * @param  {[type]} dir [description]
-		 * @return {[type]}     [description]
-		 */
+        /**
+         * 读取目录下的文件和子目录
+         * @param  {[type]} dir [description]
+         * @return {[type]}     [description]
+         */
         readDir: function (path, recursive) {
             path = path || "/";
             if (path.slice(-1) !== '/') {
@@ -302,15 +308,15 @@ module.exports = util.Class(function () {
                 }
             });
         },
-		/**
-		 * 请求数据
-		 * @param  {[type]} uri     [description]
-		 * @param  {[type]} method  [description]
-		 * @param  {[type]} data    [description]
-		 * @param  {[type]} headers [description]
-		 * @param  {[type]} options [description]
-		 * @return {[type]}         [description]
-		 */
+        /**
+         * 请求数据
+         * @param  {[type]} uri     [description]
+         * @param  {[type]} method  [description]
+         * @param  {[type]} data    [description]
+         * @param  {[type]} headers [description]
+         * @param  {[type]} options [description]
+         * @return {[type]}         [description]
+         */
         request: function (uri, method, data, headers, options) {
             uri = "/" + this.bucketname + uri;
             method = method || "GET";
@@ -327,15 +333,28 @@ module.exports = util.Class(function () {
                 'Authorization': Authorization
             });
             var deferred = util.getDefer();
+            var url = '';
+            if (this.domain == this.DEFAULT_DOMAIN) {
+                url = "https://" + this.domain + uri;
+            } else {
+                url = "http://" + this.domain + uri;
+            }
             var opts = util.extend({
-                url: "http://" + this.domain + uri,
+                url: url,
                 method: method,
                 body: data || "",
-                headers: headers
+                timeout: 10000,
+                headers: headers,
+                maxAttempts: 5,
+                retryDelay: 1000,
+                retryStrategy: request.RetryStrategies.HTTPOrNetworkError
             }, options);
             request(opts, function (error, response, body) {
-                if (error || response.statusCode !== 200) {
-                    deferred.reject(error || "statusCode: " + response.statusCode + "; body: " + body);
+                if (error || response == undefined || response.statusCode !== 200) {
+                    deferred.reject({
+                        code: (response && response.statusCode) ? response.statusCode : -1,
+                        message: error || "statusCode: " + response.statusCode + "; body: " + body
+                    });
                 } else {
                     deferred.resolve(response);
                 }
@@ -343,4 +362,4 @@ module.exports = util.Class(function () {
             return deferred.promise;
         }
     };
-});
+});

source/package.json

@@ -18,6 +18,7 @@
         "mustache": "^3.0.0",
         "mysql": "^2.16.0",
         "request": "^2.88.0",
+        "requestretry": "^3.1.0",
         "thinkjs-util": ">=0.0.1"
     }
 }

source/test/api-test.js

@@ -3,12 +3,12 @@ var config = require('../lib/config.js')
 var env = require('../lib/env.js')
 
 async function run() {
-    console.log(await api.countStudents(1, true, false));
-    let array = await api.getStudents(1, 1, 5000, true, false)
+    console.log(await api.countStudents(1, true));
+    let array = await api.getStudents(1, 1, 5000, true)
     console.log(array.length);
 }
 
-env.server = config.servers[2]
-env.loginName = 'admin-hk'
-env.password = '123456'
+env.server = config.servers[0]
+env.loginName = 'admin-test'
+env.password = 'test!@#'
 run();

source/test/image-test.js

@@ -0,0 +1,10 @@
+const image = require('../lib/image.js')()
+const upyun = require('../lib/upyun.js')
+const config = require('../lib/config.js')
+
+let client = upyun('gx-sheet', config.upyun.operator, config.upyun.password)
+image.downloadFile('/55-1402/20053A/181000885-2.jpg', '/1.jpg', {}, '/Users/luoshi', client, 'gx-sheet', false).then(() => {
+    console.log('success')
+}).catch(err => {
+    console.log(err)
+})

source/test/upyun-test.js

@@ -1,11 +1,16 @@
-const upyun = require('node-upyun-sdk')
-const env = require('../lib/env.js')
+const upyun = require('../lib/upyun.js')
 const config = require('../lib/config.js')
 
-env.server = config.servers[0]
-let client = upyun(env.server.bucket + '-sheet', config.upyun.operator, config.upyun.password)
-client.getInfo('/49-100/36431/11001490-1.jpg').then(result => {
+//env.server = config.servers[0]
+let client = upyun('gx-sheet', config.upyun.operator, config.upyun.password)
+client.getInfo('/55-1402/20053A/181000885-2.jpg').then(result => {
     console.log(result)
 }).catch(err => {
     console.log(err)
 })
+
+client.download('/55-1402/20053A/181000885-2.jpg').then(result => {
+    console.log('success:' + result.length)
+}).catch(err => {
+    console.log(err)
+})