|
@@ -5,15 +5,13 @@ const env = require('./env.js')
|
|
|
const config = require('./config.js')
|
|
|
const logger = require('./logger.js')('image.js')
|
|
|
const downloadLogger = require('./logger.js')('download')
|
|
|
-const upyun = require('./upyun.js')
|
|
|
-const PromisePool = require('./promise-pool.js')
|
|
|
const fs = require('fs')
|
|
|
const path = require('path')
|
|
|
const readline = require('readline')
|
|
|
+const request_util = require('requestretry')
|
|
|
const sizeOf = require('image-size')
|
|
|
const mustache = require('mustache')
|
|
|
const mkdirp = require('mkdirp')
|
|
|
-const moment = require('moment')
|
|
|
const gm = config.imagemagick != undefined ? require('gm').subClass({
|
|
|
imageMagick: true,
|
|
|
appPath: config.imagemagick
|
|
@@ -40,13 +38,6 @@ class executor extends EventEmitter {
|
|
|
})
|
|
|
}
|
|
|
|
|
|
- async checkFile(url, client) {
|
|
|
- let size = sizeOf(await client.download(url))
|
|
|
- if (size.width == 0 || size.height == 0) {
|
|
|
- throw 'invalid image data:' + url
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
async addWatermark(image, file, student, index, showMarker, showHeader) {
|
|
|
let fontFile = config.watermark.fontFile
|
|
|
let color = config.watermark.color
|
|
@@ -154,9 +145,29 @@ class executor extends EventEmitter {
|
|
|
})
|
|
|
}
|
|
|
|
|
|
- async downloadFile(append, remoteTemplate, localTemplate, data, dir, client, bucket, index, watermark, showMarker, showHeader) {
|
|
|
+ async downloadUrl(url) {
|
|
|
+ return new Promise((resolve, reject) => {
|
|
|
+ request_util({
|
|
|
+ url: url,
|
|
|
+ method: 'GET',
|
|
|
+ timeout: 3000,
|
|
|
+ maxAttempts: 3,
|
|
|
+ retryDelay: 500,
|
|
|
+ retryStrategy: request_util.RetryStrategies.HTTPOrNetworkError
|
|
|
+ }, function (error, response, body) {
|
|
|
+ if (response.statusCode == 200) {
|
|
|
+ resolve(body);
|
|
|
+ } else {
|
|
|
+ let message = response.statusCode + ' ' + (error || '') + (response.headers['error-info'] || '')
|
|
|
+ logger.error(message)
|
|
|
+ reject(message)
|
|
|
+ }
|
|
|
+ })
|
|
|
+ })
|
|
|
+ }
|
|
|
+
|
|
|
+ async downloadFile(type, append, url, localTemplate, data, dir, index, watermark, showMarker, showHeader) {
|
|
|
data.index = index
|
|
|
- let remote = mustache.render(remoteTemplate, data)
|
|
|
let local = path.join(dir, mustache.render(localTemplate, data))
|
|
|
mkdirp.sync(path.dirname(local))
|
|
|
|
|
@@ -165,26 +176,19 @@ class executor extends EventEmitter {
|
|
|
return Promise.resolve()
|
|
|
} else {
|
|
|
let imgData
|
|
|
- if (config.localStore != undefined && config.localStore.length > 0) {
|
|
|
- let cache = path.join(config.localStore, bucket, remote)
|
|
|
- if (fs.existsSync(cache)) {
|
|
|
- imgData = fs.readFileSync(cache)
|
|
|
- }
|
|
|
- }
|
|
|
- if (imgData == undefined) {
|
|
|
- try {
|
|
|
- imgData = await client.download(remote)
|
|
|
- } catch (err) {
|
|
|
- if (err.code === 404) {
|
|
|
- //文件不存在,记录日志并跳过
|
|
|
- downloadLogger.error('404 ' + bucket + ' ' + remote)
|
|
|
- return Promise.resolve()
|
|
|
- } else {
|
|
|
- logger.error(err)
|
|
|
- return Promise.reject(err)
|
|
|
- }
|
|
|
+ try {
|
|
|
+ imgData = await this.downloadUrl(url)
|
|
|
+ } catch (err) {
|
|
|
+ if (err.code === 404) {
|
|
|
+ //文件不存在,记录日志并跳过
|
|
|
+ downloadLogger.error('404 ' + type + ' ' + url)
|
|
|
+ return Promise.resolve()
|
|
|
+ } else {
|
|
|
+ logger.error(err)
|
|
|
+ return Promise.reject(err)
|
|
|
}
|
|
|
}
|
|
|
+
|
|
|
//是否需要添加分数水印
|
|
|
if (watermark) {
|
|
|
return this.addWatermark(imgData, local, data, index, showMarker, showHeader)
|
|
@@ -205,15 +209,8 @@ class executor extends EventEmitter {
|
|
|
}
|
|
|
|
|
|
async downloadSheet(dir, template, append, failover, watermark, showMarker, showHeader, params) {
|
|
|
- let bucket = env.server.bucketPrefix + '-sheet'
|
|
|
- let client = upyun(bucket, config.upyun.operator, config.upyun.password)
|
|
|
- if (env.server.upyunDomain && env.server.upyunDomain != '') {
|
|
|
- //局域网模式修改图片服务器地址
|
|
|
- client.setDomain(env.server.upyunDomain)
|
|
|
- }
|
|
|
-
|
|
|
try {
|
|
|
- let totalCount = await api.countStudents(env.examId, true, undefined)
|
|
|
+ let totalCount = await api.countStudents(env.examId, true)
|
|
|
this.emit('total', totalCount)
|
|
|
|
|
|
let count = 0
|
|
@@ -222,7 +219,11 @@ class executor extends EventEmitter {
|
|
|
for (;;) {
|
|
|
pageNumber++
|
|
|
|
|
|
- let array = await api.getStudents(env.examId, pageNumber, 10, true, undefined, watermark === true, watermark === true, params)
|
|
|
+ params.upload = true
|
|
|
+ params.withSheeturl = true
|
|
|
+ params.withScoreDetail = watermark === true
|
|
|
+ params.withMarkTrack = watermark === true
|
|
|
+ let array = await api.getStudents(env.examId, pageNumber, 10, params)
|
|
|
if (array == undefined || array.length == 0) {
|
|
|
break
|
|
|
}
|
|
@@ -230,8 +231,8 @@ class executor extends EventEmitter {
|
|
|
let promises = []
|
|
|
let student = array[i]
|
|
|
student.examId = env.examId
|
|
|
- for (let i = 1; i <= student.sheetCount; i++) {
|
|
|
- promises.push(this.downloadFile(append, config.imageUrl.sheet, template, student, dir, client, bucket, i, watermark, showMarker, showHeader))
|
|
|
+ for (let i = 0; i < student.sheetUrls.length; i++) {
|
|
|
+ promises.push(this.downloadFile('sheet', append, student.sheetUrls[i], template, student, dir, i + 1, watermark, showMarker, showHeader))
|
|
|
}
|
|
|
try {
|
|
|
//等待所有图片下载完毕
|
|
@@ -259,23 +260,17 @@ class executor extends EventEmitter {
|
|
|
}
|
|
|
|
|
|
async downloadPackage(dir, template, append, failover) {
|
|
|
- let bucket = env.server.bucketPrefix + '-package'
|
|
|
- let client = upyun(bucket, config.upyun.operator, config.upyun.password)
|
|
|
- if (env.server.upyunDomain && env.server.upyunDomain != '') {
|
|
|
- client.setDomain(env.server.upyunDomain)
|
|
|
- }
|
|
|
-
|
|
|
try {
|
|
|
- let array = await api.getPackages(env.examId, true)
|
|
|
+ let array = await api.getPackages(env.examId, true, true)
|
|
|
this.emit('total', array.length)
|
|
|
let count = 0
|
|
|
this.emit('count', 0)
|
|
|
for (let i = 0; i < array.length; i++) {
|
|
|
let p = array[i]
|
|
|
p.examId = env.examId
|
|
|
- for (let i = 1; i <= p.picCount; i++) {
|
|
|
+ for (let i = 0; i < p.urls.length; i++) {
|
|
|
try {
|
|
|
- await this.downloadFile(append, config.imageUrl.package, template, p, dir, client, bucket, i)
|
|
|
+ await this.downloadFile('package', append, p.urls[i], template, p, dir, i + 1)
|
|
|
} catch (err) {
|
|
|
//判断是否异常终止
|
|
|
if (failover) {
|
|
@@ -297,124 +292,8 @@ class executor extends EventEmitter {
|
|
|
this.emit('error', error)
|
|
|
}
|
|
|
}
|
|
|
-
|
|
|
- async checkSlice(dir, concurrent) {
|
|
|
- let bucket = env.server.bucketPrefix + '-slice'
|
|
|
- let client = upyun(bucket, config.upyun.operator, config.upyun.password)
|
|
|
- if (env.server.upyunDomain && env.server.upyunDomain != '') {
|
|
|
- //局域网模式修改图片服务器地址
|
|
|
- client.setDomain(env.server.upyunDomain)
|
|
|
- }
|
|
|
-
|
|
|
- try {
|
|
|
- let logFile = path.join(dir, 'result.txt')
|
|
|
- fs.writeFileSync(logFile, moment().format('YYYY-MM-DD HH:mm:ss') + ', examId=' + env.examId + '\r\n')
|
|
|
-
|
|
|
- let totalCount = await api.countStudents(env.examId, true, false)
|
|
|
- this.emit('total', totalCount)
|
|
|
-
|
|
|
- let self = this
|
|
|
- let count = 0
|
|
|
- let pageNumber = 0
|
|
|
- let pool = PromisePool.create(concurrent, function(student) {
|
|
|
- student.examId = env.examId
|
|
|
- student.promises = []
|
|
|
- for (let i = 1; i <= student.sliceCount; i++) {
|
|
|
- student.index = i
|
|
|
- let url = mustache.render(config.imageUrl.slice, student)
|
|
|
- student.promises.push(new Promise(resolved => {
|
|
|
- self.checkFile(url, client).then(() => {
|
|
|
- //fs.appendFileSync(logFile, url + ': success\r\n')
|
|
|
- }).catch(() => {
|
|
|
- fs.appendFileSync(logFile, url + ': error\r\n')
|
|
|
- }).finally(() => {
|
|
|
- resolved()
|
|
|
- })
|
|
|
- }))
|
|
|
- }
|
|
|
- return Promise.all(student.promises)
|
|
|
- })
|
|
|
- pool.on('count', function(offset) {
|
|
|
- self.emit('count', count + offset)
|
|
|
- })
|
|
|
-
|
|
|
- this.emit('count', 0)
|
|
|
- for (;;) {
|
|
|
- pageNumber++
|
|
|
- let array = await api.getStudents(env.examId, pageNumber, 200, true, false, false, false)
|
|
|
- if (array == undefined || array.length == 0) {
|
|
|
- break
|
|
|
- }
|
|
|
- await pool.start(array)
|
|
|
- count += array.length
|
|
|
- }
|
|
|
- this.emit('finish')
|
|
|
- fs.appendFileSync(logFile, moment().format('YYYY-MM-DD HH:mm:ss') + ', examId=' + env.examId)
|
|
|
- } catch (error) {
|
|
|
- logger.error('check slice error:' + error)
|
|
|
- logger.error(error)
|
|
|
- this.emit('error', error)
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- async checkSliceSerial(dir) {
|
|
|
- let bucket = env.server.bucketPrefix + '-slice'
|
|
|
- let client = upyun(bucket, config.upyun.operator, config.upyun.password)
|
|
|
- if (env.server.upyunDomain && env.server.upyunDomain != '') {
|
|
|
- //局域网模式修改图片服务器地址
|
|
|
- client.setDomain(env.server.upyunDomain)
|
|
|
- }
|
|
|
-
|
|
|
- try {
|
|
|
- let logFile = path.join(dir, 'result.txt')
|
|
|
- fs.writeFileSync(logFile, moment().format('YYYY-MM-DD HH:mm:ss') + ', examId=' + env.examId + '\r\n')
|
|
|
-
|
|
|
- let totalCount = await api.countStudents(env.examId, true, false)
|
|
|
- this.emit('total', totalCount)
|
|
|
-
|
|
|
- let self = this
|
|
|
- let count = 0
|
|
|
- let pageNumber = 0
|
|
|
- this.emit('count', 0)
|
|
|
- for (;;) {
|
|
|
- pageNumber++
|
|
|
-
|
|
|
- let array = await api.getStudents(env.examId, pageNumber, 100, true, false, false, false)
|
|
|
- if (array == undefined || array.length == 0) {
|
|
|
- break
|
|
|
- }
|
|
|
- for (let i = 0; i < array.length; i++) {
|
|
|
- let student = array[i]
|
|
|
- student.examId = env.examId
|
|
|
- student.promises = []
|
|
|
- for (let i = 1; i <= student.sliceCount; i++) {
|
|
|
- student.index = i
|
|
|
- let url = mustache.render(config.imageUrl.slice, student)
|
|
|
- student.promises.push(new Promise(resolved => {
|
|
|
- self.checkFile(url, client).then(() => {
|
|
|
- //fs.appendFileSync(logFile, url + ': success\r\n')
|
|
|
- }).catch(() => {
|
|
|
- fs.appendFileSync(logFile, url + ': error\r\n')
|
|
|
- }).finally(() => {
|
|
|
- resolved()
|
|
|
- })
|
|
|
- }))
|
|
|
- }
|
|
|
- await Promise.all(student.promises)
|
|
|
- count++
|
|
|
- this.emit('count', count)
|
|
|
- }
|
|
|
- }
|
|
|
- this.emit('finish')
|
|
|
- fs.appendFileSync(logFile, moment().format('YYYY-MM-DD HH:mm:ss') + ', examId=' + env.examId)
|
|
|
- } catch (error) {
|
|
|
- logger.error('check slice error:' + error)
|
|
|
- logger.error(error)
|
|
|
- this.emit('error', error)
|
|
|
- }
|
|
|
- }
|
|
|
}
|
|
|
|
|
|
-module.exports = function() {
|
|
|
+module.exports = function () {
|
|
|
return new executor()
|
|
|
}
|