fix

src/main/java/cn/com/qmth/scancloud/tools/config/SysProperty.java

@@ -29,7 +29,8 @@ public class SysProperty {
 
     // 默认线程数
     public static Integer THREAD_SIZE;
-    
+    // 默认每批处理数
+    public static Integer BATCH_SIZE;
     //年度后两位
     public static String YEAR;
     
@@ -82,7 +83,7 @@ public class SysProperty {
         TEMPLATE_SEPARATOR = templateSeparator;
     }
 
-    @Value("${scan.tool.thread.size:3}")
+    @Value("${scan.tool.thread.size:4}")
     public void threadSize(Integer threadSize) {
         THREAD_SIZE = threadSize;
     }
@@ -125,4 +126,9 @@ public class SysProperty {
     public void absentImport(String absentImport) {
         ABSENT_IMPORT = absentImport;
     }
+    @Value("${scan.tool.batch.size:5000}")
+    public void batchSize(Integer batchSize) {
+        BATCH_SIZE = batchSize;
+    }
+    
 }

src/main/java/cn/com/qmth/scancloud/tools/config/TaskConfig.java

@@ -2,6 +2,8 @@ package cn.com.qmth.scancloud.tools.config;
 
 import cn.com.qmth.scancloud.tools.enums.TaskType;
 import cn.com.qmth.scancloud.tools.service.AbstractTask;
+import cn.com.qmth.scancloud.tools.utils.SpringContextHolder;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -18,7 +20,7 @@ public class TaskConfig {
 
         AbstractTask task;
         try {
-            task = (AbstractTask) taskType.getImpl().newInstance();
+            task = (AbstractTask)SpringContextHolder.getBean(taskType.getImpl());
         } catch (Exception e) {
             log.error(e.getMessage(), e);
             return;

src/main/java/cn/com/qmth/scancloud/tools/multithread/AopTargetUtils.java

@@ -0,0 +1,54 @@
+package cn.com.qmth.scancloud.tools.multithread;
+
+import org.springframework.aop.framework.AdvisedSupport;
+import org.springframework.aop.framework.AopProxy;
+import org.springframework.aop.support.AopUtils;
+
+import java.lang.reflect.Field;
+
+public class AopTargetUtils {
+
+	public static Object getTarget(Object obj) {
+		if (!AopUtils.isAopProxy(obj)) {
+			return obj;
+		}
+		try {
+
+			// 判断是jdk还是cglib代理
+			if (AopUtils.isJdkDynamicProxy(obj)) {
+				obj = getJdkDynamicProxyTargetObject(obj);
+			} else {
+				obj = getCglibDynamicProxyTargetObject(obj);
+			}
+
+		} catch (Exception e) {
+
+		}
+		return obj;
+
+	}
+
+	private static Object getCglibDynamicProxyTargetObject(Object obj) throws Exception {
+		Field h = obj.getClass().getDeclaredField("CGLIB$CALLBACK_0");
+		h.setAccessible(true);
+
+		Object dynamicAdvisedInterceptor = h.get(obj);
+		Field advised = dynamicAdvisedInterceptor.getClass().getDeclaredField("advised");
+		advised.setAccessible(true);
+		Object target = ((AdvisedSupport) advised.get(dynamicAdvisedInterceptor)).getTargetSource().getTarget();
+		return target;
+	}
+
+	private static Object getJdkDynamicProxyTargetObject(Object obj) throws Exception {
+
+		Field h = obj.getClass().getSuperclass().getDeclaredField("h");
+		h.setAccessible(true);
+
+		AopProxy aopProxy = (AopProxy) h.get(obj);
+		Field advised = aopProxy.getClass().getDeclaredField("advised");
+		advised.setAccessible(true);
+		Object target = ((AdvisedSupport) advised.get(aopProxy)).getTargetSource().getTarget();
+		return target;
+
+	}
+}

src/main/java/cn/com/qmth/scancloud/tools/multithread/Basket.java

@@ -0,0 +1,133 @@
+package cn.com.qmth.scancloud.tools.multithread;
+
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import cn.com.qmth.scancloud.tools.utils.Calculator;
+import cn.com.qmth.scancloud.tools.utils.StatusException;
+
+public class Basket {
+
+    private Integer total=0;
+
+    private AtomicInteger process=new AtomicInteger(0);
+
+    /**
+     * 数据阻塞队列
+     */
+    private BlockingQueue<Object> queue;
+
+    /**
+     * 多线程计数器，子线程都结束后主线程才继续执行
+     */
+    private CountDownLatch endGate;
+
+    /**
+     * 消费者数量
+     */
+    private int consumerCount;
+
+    /**
+     * 判断线程执行是否有出错，生产者、消费者出错都需要修改此值为true
+     */
+    private boolean isExcuteError = false;
+
+    public Basket(int consumerCount) {
+        this.consumerCount = consumerCount;
+        queue = new ArrayBlockingQueue<Object>(consumerCount * 2);
+        endGate = new CountDownLatch(consumerCount);
+    }
+
+    /**
+     * 生产数据，不采用put方法防止消费线程全部异常后生产线程阻塞
+     * 
+     * @param value
+     * @throws InterruptedException
+     */
+    protected void offer(final Object value) throws InterruptedException {
+        if (isExcuteError) {
+            throw new StatusException("线程异常");
+        } else {
+            boolean ret = queue.offer(value, 1, TimeUnit.MINUTES);
+            if (!ret) {
+                this.offer(value);
+            }
+        }
+    }
+
+    /**
+     * 消费数据，不采用take方法防止生产线程全部异常后消费线程阻塞
+     * 
+     * @return
+     * @throws InterruptedException
+     */
+    protected Object consume() throws InterruptedException {
+        if (isExcuteError) {
+            return new EndObject();
+        } else {
+            Object ob = queue.poll(1, TimeUnit.MINUTES);
+            if (ob == null) {
+                return this.consume();
+            } else {
+                return ob;
+            }
+        }
+    }
+
+    protected void endGateReset() {
+        endGate = new CountDownLatch(consumerCount);
+    }
+
+    protected void await() throws InterruptedException {
+        endGate.await();
+    }
+
+    protected void countDown() {
+        endGate.countDown();
+    }
+
+    protected boolean isExcuteError() {
+        return isExcuteError;
+    }
+
+    protected void setExcuteError(boolean isExcuteError) {
+        this.isExcuteError = isExcuteError;
+    }
+
+    protected int getConsumerCount() {
+        return consumerCount;
+    }
+
+    protected void setConsumerCount(int consumerCount) {
+        this.consumerCount = consumerCount;
+    }
+
+    public Integer getTotal() {
+        return total;
+    }
+
+    protected void setTotal(Integer total) {
+        this.total = total;
+    }
+
+    public AtomicInteger getProcess() {
+        return process;
+    }
+
+    protected void updateProcess(int add) {
+        process.addAndGet(add);
+    }
+    
+    public String getProgress() {
+        if(total==0) {
+           return "0%"; 
+        }
+        Double d=Calculator.divide(process.doubleValue(), total.doubleValue(), 4);
+        Double f=Calculator.multiply(d,100);
+        return f+"%";
+    }
+
+}

src/main/java/cn/com/qmth/scancloud/tools/multithread/BatchConsumer.java

@@ -0,0 +1,54 @@
+package cn.com.qmth.scancloud.tools.multithread;
+
+import java.util.concurrent.locks.LockSupport;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class BatchConsumer<T>  extends Thread{
+	private static final Logger LOG = LoggerFactory.getLogger(BatchConsumer.class);
+	private Basket basket;
+	
+	public BatchConsumer() {
+	}
+	@SuppressWarnings("unchecked")
+	@Override
+	public void run() {
+		try {
+			for (;;) {
+				//先判断是否有异常结束
+				if(basket.isExcuteError()) {
+					break;
+				}
+				//取消费数据
+				Object o= basket.consume();
+				//判断消费数据是否是结束
+				if(o instanceof EndObject) {
+					basket.countDown();
+					break;
+				}
+				if(o instanceof ParkObject) {
+					basket.countDown();
+					LockSupport.park();
+				}else {
+					T t=(T)o;
+					//消费数据实现
+					consume(t);
+				}
+			}
+		} catch (Exception e) {
+			basket.setExcuteError(true);
+			LOG.error("消费线程处理出错",e);
+		} finally {
+			basket.countDown();
+		}
+	}
+	public abstract void consume(T t);
+	protected Basket getBasket() {
+		return basket;
+	}
+	protected void setBasket(Basket basket) {
+		this.basket = basket;
+	}
+	
+}

src/main/java/cn/com/qmth/scancloud/tools/multithread/BatchProducer.java

@@ -0,0 +1,170 @@
+package cn.com.qmth.scancloud.tools.multithread;
+
+import java.lang.reflect.ParameterizedType;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.LockSupport;
+
+import org.apache.commons.collections4.CollectionUtils;
+
+import cn.com.qmth.scancloud.tools.utils.SpringContextHolder;
+import cn.com.qmth.scancloud.tools.utils.StatusException;
+
+/**
+ * @param <T> 生产消费对象
+ * @param <C> 消费线程类
+ */
+public abstract class BatchProducer<T, C extends BatchConsumer<T>> {
+
+    private List<BatchConsumer<T>> consumers;
+
+    private Basket basket;
+
+    public void startDispose(int consumerCount) throws InterruptedException {
+        startDispose(consumerCount, null);
+    }
+
+    /**
+     * 处理开始方法
+     *
+     * @param consumerCount 消费线程数
+     * @param param         生产者业务参数
+     */
+    public void startDispose(int consumerCount, Map<String, Object> param) throws InterruptedException {
+        // 启动消费者
+        try {
+            startConsumer(consumerCount);
+            for (; ; ) {
+                List<T> dtos = findOneBatchData(param);
+                if (CollectionUtils.isEmpty(dtos)) {
+                    // 拿不到数据，结束消费
+                    endConsumer();
+                    break;
+                }
+                // 开始处理一轮
+                disposeBatch(dtos);
+                //重置计数器
+                basket.endGateReset();
+                // 唤醒消费者
+                unParkConsumer();
+            }
+        } catch (StatusException e) {
+            // 获取异常时发送异常结束信息
+            endConsumerAsError();
+            throw e;
+        } catch (Exception e) {
+            // 获取异常时发送异常结束信息
+            endConsumerAsError();
+            throw new StatusException("处理失败", e);
+        }
+    }
+
+    /**
+     * 获取一个批次数据，都被消费处理完毕后才再取下一批
+     *
+     * @param param
+     * @return
+     */
+    public abstract List<T> findOneBatchData(Map<String, Object> param);
+
+    @SuppressWarnings("unchecked")
+    private void startConsumer(int consumerCount) {
+        if (consumerCount <= 0) {
+            consumerCount = 1;
+        }
+        ParameterizedType pt = (ParameterizedType) this.getClass().getGenericSuperclass();
+        Class<C> clazz = (Class<C>) pt.getActualTypeArguments()[1];
+        consumers = new ArrayList<>();
+        this.basket = new Basket(consumerCount);
+        // 启动消费者
+        int count = basket.getConsumerCount();
+        for (int i = 0; i < count; i++) {
+            BatchConsumer<T> co = SpringContextHolder.getBean(clazz);
+            co.setBasket(basket);
+            co.start();
+            consumers.add((BatchConsumer<T>) AopTargetUtils.getTarget(co));
+        }
+    }
+
+    private void disposeBatch(List<T> dtos) throws InterruptedException {
+        // 生产数据
+        for (T t : dtos) {
+            offer(t);
+        }
+        // 发送一轮结束信息
+        parkConsumer();
+        // 等待子线程结束
+        await();
+        // 判断子线程是否正常结束
+        if (basket.isExcuteError()) {
+            throw new StatusException("处理失败,线程异常");
+        }
+    }
+
+    /**
+     * 出异常后修改标识
+     */
+    private void endConsumerAsError() {
+        basket.setExcuteError(true);
+    }
+
+    /**
+     * 正常结束消费者
+     *
+     * @throws InterruptedException
+     */
+    private void endConsumer() throws InterruptedException {
+        int count = basket.getConsumerCount();
+        EndObject eo = new EndObject();
+        for (int i = 0; i < count; i++) {
+            basket.offer(eo);
+        }
+
+    }
+
+    /**
+     * 正常暂停消费者
+     *
+     * @throws InterruptedException
+     */
+    private void parkConsumer() throws InterruptedException {
+        int count = basket.getConsumerCount();
+        ParkObject eo = new ParkObject();
+        for (int i = 0; i < count; i++) {
+            basket.offer(eo);
+        }
+
+    }
+
+    /**
+     * 正常唤醒消费者
+     *
+     * @throws InterruptedException
+     */
+    private void unParkConsumer() {
+        for (BatchConsumer<T> c : consumers) {
+            LockSupport.unpark(c);
+        }
+    }
+
+    /**
+     * 生产数据
+     *
+     * @param ob
+     * @throws InterruptedException
+     */
+    private void offer(Object ob) throws InterruptedException {
+        basket.offer(ob);
+    }
+
+    /**
+     * 等待所有消费者结束
+     *
+     * @throws InterruptedException
+     */
+    private void await() throws InterruptedException {
+        basket.await();
+    }
+
+}

src/main/java/cn/com/qmth/scancloud/tools/multithread/Consumer.java

@@ -0,0 +1,57 @@
+package cn.com.qmth.scancloud.tools.multithread;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class Consumer<T> extends Thread {
+
+    private static final Logger LOG = LoggerFactory.getLogger(Consumer.class);
+
+    private Basket basket;
+
+    public Consumer() {
+    }
+
+    @Override
+    public void run() {
+        try {
+            while (true) {
+                // 先判断是否有异常结束
+                if (basket.isExcuteError()) {
+                    break;
+                }
+                // 取消费数据
+                Object o = basket.consume();
+                // 判断消费数据是否是结束
+                if (o instanceof EndObject) {
+                    break;
+                }
+                @SuppressWarnings("unchecked")
+                T t = (T) o;
+                // 消费数据实现
+                int disposeCount = consume(t);
+                if (basket.getTotal() > 0) {
+                    basket.updateProcess(disposeCount);
+                    processInfo();
+                }
+            }
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+            basket.setExcuteError(true);
+        } finally {
+            basket.countDown();
+        }
+    }
+
+    protected abstract int consume(T t);
+
+    protected void setBasket(Basket basket) {
+        this.basket = basket;
+    }
+    protected Basket getBasket() {
+        return this.basket;
+    }
+    protected void processInfo() {
+        LOG.info("处理进度" + basket.getProgress());
+    }
+}

src/main/java/cn/com/qmth/scancloud/tools/multithread/EndObject.java

@@ -0,0 +1,10 @@
+package cn.com.qmth.scancloud.tools.multithread;
+
+/**
+ * 消费结束标识对象
+ * @author xiatian
+ *
+ */
+public class EndObject {
+
+}

src/main/java/cn/com/qmth/scancloud/tools/multithread/ParkObject.java

@@ -0,0 +1,10 @@
+package cn.com.qmth.scancloud.tools.multithread;
+
+/**
+ * 消费一轮完毕标识对象
+ * @author xiatian
+ *
+ */
+public class ParkObject {
+
+}

src/main/java/cn/com/qmth/scancloud/tools/multithread/Producer.java

@@ -0,0 +1,140 @@
+package cn.com.qmth.scancloud.tools.multithread;
+
+import java.lang.reflect.ParameterizedType;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import cn.com.qmth.scancloud.tools.utils.SpringContextHolder;
+import cn.com.qmth.scancloud.tools.utils.StatusException;
+
+public abstract class Producer<T, C extends Consumer<T>> {
+    private static final Logger LOG = LoggerFactory.getLogger(Producer.class);
+    private Basket basket;
+
+    /**
+     * 消费线程class
+     */
+    private List<Consumer<T>> consumers;
+
+    public void startDispose(int consumerCount){
+        startDispose(consumerCount, null,0);
+    }
+
+    /**
+     * 处理开始方法
+     */
+    public void startDispose(int consumerCount, Map<String, Object> param) {
+        startDispose(consumerCount, param,0);
+    }
+    public void startDispose(int consumerCount,int total) {
+        startDispose(consumerCount, null,total);
+    }
+    public void startDispose(int consumerCount, Map<String, Object> param,int total) {
+        // 启动消费者
+        startConsumer(consumerCount,total);
+        // 开始处理
+        dispose(param);
+    }
+    @SuppressWarnings("unchecked")
+    private void startConsumer(int consumerCount,int total) {
+        if (consumerCount <= 0) {
+            consumerCount = 1;
+        }
+        ParameterizedType pt = (ParameterizedType) this.getClass().getGenericSuperclass();
+        Class<C> clazz = (Class<C>) pt.getActualTypeArguments()[1];
+        consumers = new ArrayList<>();
+        this.basket = new Basket(consumerCount);
+        basket.setTotal(total);
+        // 启动消费者
+        int count = basket.getConsumerCount();
+        for (int i = 0; i < count; i++) {
+            Consumer<T> co = SpringContextHolder.getBean(clazz);
+            co.setBasket(basket);
+            co.start();
+            consumers.add((Consumer<T>) AopTargetUtils.getTarget(co));
+        }
+    }
+
+    private void dispose(Map<String, Object> param) {
+        try {
+            // 生产数据
+            int index = 0;
+            for (;;) {
+                T dto = findData(param, index);
+                if (dto == null) {
+                    // 拿不到数据，结束消费
+                    break;
+                }
+                offer(dto);
+                index++;
+            }
+            // 发送生产结束信息
+            endConsumer();
+            // 等待子线程结束
+            await();
+            // 判断子线程是否正常结束
+            if (basket.isExcuteError()) {
+                throw new StatusException("处理失败,线程异常");
+            }
+        } catch (StatusException e) {
+            LOG.error(e.getMessage(),e);
+            // 获取异常时发送异常结束信息
+            endConsumerAsError();
+            throw e;
+        } catch (Exception e) {
+            LOG.error(e.getMessage(),e);
+            // 获取异常时发送异常结束信息
+            endConsumerAsError();
+            throw new StatusException("处理失败", e);
+        }
+    }
+
+    /**
+     * 出异常后修改标识
+     * 
+     */
+    private void endConsumerAsError() {
+        basket.setExcuteError(true);
+    }
+
+    /**
+     * 正常结束消费者
+     * 
+     * @throws InterruptedException
+     */
+    private void endConsumer() throws InterruptedException {
+        int count = basket.getConsumerCount();
+        EndObject eo = new EndObject();
+        for (int i = 0; i < count; i++) {
+            basket.offer(eo);
+        }
+
+    }
+
+    /**
+     * 生产数据
+     * 
+     * @param ob
+     * @throws InterruptedException
+     */
+    private void offer(Object ob) throws InterruptedException {
+        synchronized (basket) {
+            basket.offer(ob);
+        }
+    }
+
+    /**
+     * 等待所有消费者结束
+     * 
+     * @throws InterruptedException
+     */
+    private void await() throws InterruptedException {
+        basket.await();
+    }
+
+    protected abstract T findData(Map<String, Object> param, int index);
+}

src/main/java/cn/com/qmth/scancloud/tools/multithread/consumer/ExamStudentImportConsumer.java

@@ -0,0 +1,35 @@
+package cn.com.qmth.scancloud.tools.multithread.consumer;
+
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Service;
+
+import cn.com.qmth.scancloud.tools.model.ExamStudent;
+import cn.com.qmth.scancloud.tools.multithread.Consumer;
+import cn.com.qmth.scancloud.tools.service.ExamStudentImportService;
+import cn.com.qmth.scancloud.tools.service.impl.ExamStudentImportTask;
+
+@Scope("prototype")
+@Service
+public class ExamStudentImportConsumer extends Consumer<List<ExamStudent>> {
+    private static final Logger log = LoggerFactory.getLogger(ExamStudentImportTask.class);
+    @Autowired
+    private ExamStudentImportService ExamStudentImportService;
+    
+    @Override
+    public int consume(List<ExamStudent> dto) {
+        ExamStudentImportService.studentImport(dto);
+        return dto.size();
+    }
+
+    @Override
+    protected void processInfo() {
+        log.info("已处理数：{} 进度：{} ", this.getBasket().getProcess().intValue(), this.getBasket().getProgress());
+    }
+
+
+}

src/main/java/cn/com/qmth/scancloud/tools/multithread/producer/ExamStudentImportProducer.java

@@ -0,0 +1,31 @@
+package cn.com.qmth.scancloud.tools.multithread.producer;
+
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.stereotype.Service;
+
+import cn.com.qmth.scancloud.tools.model.ExamStudent;
+import cn.com.qmth.scancloud.tools.multithread.Producer;
+import cn.com.qmth.scancloud.tools.multithread.consumer.ExamStudentImportConsumer;
+
+@Service
+public class ExamStudentImportProducer extends Producer<List<ExamStudent>, ExamStudentImportConsumer> {
+
+    @SuppressWarnings("unchecked")
+    @Override
+    protected List<ExamStudent> findData(Map<String, Object> param, int index) {
+        Integer batchSize=(Integer)param.get("batchSize");
+        List<ExamStudent> ret=(List<ExamStudent>)param.get("data");
+        int start=index*batchSize;
+        if(start>=ret.size()) {
+            return null;
+        }
+        int end=(index+1)*batchSize;
+        if(end>ret.size()) {
+            end=ret.size();
+        }
+        return ret.subList(start, end);
+    }
+
+}

src/main/java/cn/com/qmth/scancloud/tools/service/ExamStudentImportService.java

@@ -0,0 +1,24 @@
+package cn.com.qmth.scancloud.tools.service;
+
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import cn.com.qmth.scancloud.tools.config.SysProperty;
+import cn.com.qmth.scancloud.tools.model.ExamStudent;
+import cn.com.qmth.scancloud.tools.utils.HttpHelper;
+import cn.com.qmth.scancloud.tools.utils.JsonHelper;
+
+@Service
+public class ExamStudentImportService {
+
+    private static String url = SysProperty.SCAN_SERVER_URL + "/api/tool/import/exam/student";
+
+    public void studentImport(List<ExamStudent> list) {
+
+        String json = JsonHelper.toJson(list);
+        HttpHelper.post(url, json);
+
+    }
+
+}

src/main/java/cn/com/qmth/scancloud/tools/service/impl/AbsentImportTask.java

@@ -10,6 +10,7 @@ import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
 
 import cn.com.qmth.scancloud.tools.config.SysProperty;
 import cn.com.qmth.scancloud.tools.enums.TaskType;
@@ -24,6 +25,7 @@ import cn.com.qmth.scancloud.tools.utils.StatusException;
 /**
  * 缺考导入
  */
+@Service
 public class AbsentImportTask extends AbstractTask {
 
     private static final Logger log = LoggerFactory.getLogger(AbsentImportTask.class);

src/main/java/cn/com/qmth/scancloud/tools/service/impl/CourseImportTask.java

@@ -8,6 +8,7 @@ import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
 
 import cn.com.qmth.scancloud.tools.config.SysProperty;
 import cn.com.qmth.scancloud.tools.enums.TaskType;
@@ -22,6 +23,7 @@ import cn.com.qmth.scancloud.tools.utils.StatusException;
 /**
  * 课程导入
  */
+@Service
 public class CourseImportTask extends AbstractTask {
 
     private static final Logger log = LoggerFactory.getLogger(CourseImportTask.class);

src/main/java/cn/com/qmth/scancloud/tools/service/impl/ExamImportTask.java

@@ -9,6 +9,7 @@ import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
 
 import cn.com.qmth.scancloud.tools.config.SysProperty;
 import cn.com.qmth.scancloud.tools.enums.ExamMode;
@@ -24,6 +25,7 @@ import cn.com.qmth.scancloud.tools.utils.StatusException;
 /**
  * 考试导入
  */
+@Service
 public class ExamImportTask extends AbstractTask {
 
     private static final Logger log = LoggerFactory.getLogger(ExamImportTask.class);

src/main/java/cn/com/qmth/scancloud/tools/service/impl/ExamStudentCleanTask.java

@@ -7,10 +7,12 @@ import cn.com.qmth.scancloud.tools.utils.HttpHelper;
 import cn.com.qmth.scancloud.tools.utils.StatusException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
 
 /**
  * 考生清理
  */
+@Service
 public class ExamStudentCleanTask extends AbstractTask {
 
     private static final Logger log = LoggerFactory.getLogger(ExamStudentCleanTask.class);

src/main/java/cn/com/qmth/scancloud/tools/service/impl/ExamStudentCountTask.java

@@ -8,6 +8,7 @@ import cn.com.qmth.scancloud.tools.utils.JsonHelper;
 import cn.com.qmth.scancloud.tools.utils.StatusException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -15,6 +16,7 @@ import java.util.Map;
 /**
  * 考生数量检查
  */
+@Service
 public class ExamStudentCountTask extends AbstractTask {
 
     private static final Logger log = LoggerFactory.getLogger(ExamStudentCountTask.class);

src/main/java/cn/com/qmth/scancloud/tools/service/impl/ExamStudentImportTask.java

@@ -2,8 +2,10 @@ package cn.com.qmth.scancloud.tools.service.impl;
 
 import java.io.File;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.regex.Pattern;
 
@@ -11,26 +13,29 @@ import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
 
 import cn.com.qmth.scancloud.tools.config.SysProperty;
 import cn.com.qmth.scancloud.tools.enums.TaskType;
 import cn.com.qmth.scancloud.tools.model.ExamStudent;
+import cn.com.qmth.scancloud.tools.multithread.producer.ExamStudentImportProducer;
 import cn.com.qmth.scancloud.tools.service.AbstractTask;
 import cn.com.qmth.scancloud.tools.service.CommonService;
 import cn.com.qmth.scancloud.tools.utils.FileHelper;
-import cn.com.qmth.scancloud.tools.utils.HttpHelper;
-import cn.com.qmth.scancloud.tools.utils.JsonHelper;
 import cn.com.qmth.scancloud.tools.utils.StatusException;
 
 /**
  * 考生导入
  */
+@Service
 public class ExamStudentImportTask extends AbstractTask {
 
     private static final Logger log = LoggerFactory.getLogger(ExamStudentImportTask.class);
 
     private static Pattern examNumberRex = Pattern.compile("^[0-9]{15}$");
-
+    @Autowired
+    private ExamStudentImportProducer examStudentImportProducer;
     @Override
     protected String getTaskName() {
         return TaskType.EXAM_STUDENT_IMPORT.getTitle();
@@ -90,29 +95,12 @@ public class ExamStudentImportTask extends AbstractTask {
         if (!allExist) {
             throw new StatusException("科目信息有误！");
         }
-
-        // 分批保存
-        int batchSize = 5000;
-        List<ExamStudent> batchList = new ArrayList<>();
-        String url = SysProperty.SCAN_SERVER_URL + "/api/tool/import/exam/student";
-        for (int n = 0; n < list.size(); n++) {
-            batchList.add(list.get(n));
-
-            if (batchList.size() % batchSize == 0) {
-                String json = JsonHelper.toJson(batchList);
-                String result = HttpHelper.post(url, json);
-                batchList.clear();
-
-                float rate = (n + 1) * 100f / list.size();
-                log.info("已处理数：{} 进度：{}% {}", n + 1, rate, result);
-            }
-        }
-
-        if (CollectionUtils.isNotEmpty(batchList)) {
-            String json = JsonHelper.toJson(batchList);
-            String result = HttpHelper.post(url, json);
-            log.info("已处理数：{} 进度：100% {}", list.size(), result);
-        }
+        Map<String, Object> param = new HashMap<>();
+        param.put("data", list);
+        Integer threadCount=SysProperty.THREAD_SIZE;
+        Integer batchSize=SysProperty.BATCH_SIZE;
+        param.put("batchSize", batchSize);
+        examStudentImportProducer.startDispose(threadCount,param, list.size());
     }
 
     private ExamStudent parseValues(int index, String line, Set<String> examNumbers) {

src/main/java/cn/com/qmth/scancloud/tools/service/impl/ObjectiveQuestionExportTask.java

@@ -17,6 +17,7 @@ import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.qmth.boot.tools.excel.ExcelReader;
@@ -37,6 +38,7 @@ import cn.com.qmth.scancloud.tools.utils.StatusException;
 /**
  * 考生扫描结果和评卷数据导出
  */
+@Service
 public class ObjectiveQuestionExportTask extends AbstractTask {
 
     private static final Logger log = LoggerFactory.getLogger(ObjectiveQuestionExportTask.class);

src/main/java/cn/com/qmth/scancloud/tools/service/impl/ScanImageCheckTask.java

@@ -11,6 +11,7 @@ import com.fasterxml.jackson.databind.JsonNode;
 import org.apache.commons.collections4.CollectionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
 
 import java.util.HashMap;
 import java.util.List;
@@ -23,6 +24,7 @@ import java.util.concurrent.Executors;
 /**
  * 扫描图片检查&修复
  */
+@Service
 public class ScanImageCheckTask extends AbstractTask {
 
     private static final Logger log = LoggerFactory.getLogger(ScanImageCheckTask.class);

src/main/java/cn/com/qmth/scancloud/tools/service/impl/StructImportTask.java

@@ -14,6 +14,7 @@ import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
 
 import com.qmth.boot.tools.excel.ExcelReader;
 import com.qmth.boot.tools.excel.enums.ExcelType;
@@ -31,6 +32,7 @@ import cn.com.qmth.scancloud.tools.utils.StatusException;
 /**
  * 缺考导入
  */
+@Service
 public class StructImportTask extends AbstractTask {
 
     private static final Logger log = LoggerFactory.getLogger(StructImportTask.class);

src/main/java/cn/com/qmth/scancloud/tools/service/impl/UserImportTask.java

@@ -8,6 +8,7 @@ import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
 
 import cn.com.qmth.scancloud.tools.config.SysProperty;
 import cn.com.qmth.scancloud.tools.enums.RoleType;
@@ -22,6 +23,7 @@ import cn.com.qmth.scancloud.tools.utils.StatusException;
 /**
  * 管理员用户导入
  */
+@Service
 public class UserImportTask extends AbstractTask {
 
     private static final Logger log = LoggerFactory.getLogger(UserImportTask.class);

src/main/java/cn/com/qmth/scancloud/tools/utils/Calculator.java

@@ -0,0 +1,168 @@
+package cn.com.qmth.scancloud.tools.utils;
+
+import java.math.BigDecimal;
+import java.util.List;
+
+import org.apache.commons.collections4.CollectionUtils;
+
+/**
+ * 计算器
+ *
+ * @author
+ * @date 2019年7月30日
+ * @Copyright (c) 2018-? http://qmth.com.cn All Rights Reserved.
+ */
+public class Calculator {
+
+    /**
+     * 加法 保留两位小数
+     * 
+     * @param v1
+     * @param v2
+     * @return
+     */
+    public static double add(double v1, double v2) {
+        return add(v1, v2, 2);
+
+    }
+
+    /**
+     * 加法 保留指定位小数
+     * 
+     * @param v1
+     * @param v2
+     * @param len
+     * @return
+     */
+    public static double add(double v1, double v2, int len) {
+        BigDecimal b1 = new BigDecimal(v1);
+        BigDecimal b2 = new BigDecimal(v2);
+        return b1.add(b2).setScale(len, BigDecimal.ROUND_HALF_UP).doubleValue();
+
+    }
+    
+    /** 加法 保留指定位小数
+     * @param ds
+     * @param len
+     * @return
+     */
+    public static double add(List<Double> ds, int len) {
+        if(CollectionUtils.isEmpty(ds)) {
+            throw new StatusException("数组为空");
+        }
+        BigDecimal ret = new BigDecimal(0.0);
+        for(Double d:ds) {
+            if(d==null) {
+                throw new StatusException("数组元素为空");
+            }
+            ret=ret.add(new BigDecimal(d));
+        }
+        return ret.setScale(len, BigDecimal.ROUND_HALF_UP).doubleValue();
+
+    }
+
+    /**
+     * 减法 保留两位小数
+     * 
+     * @param v1
+     * @param v2
+     * @return
+     */
+    public static double subtract(double v1, double v2) {
+        return subtract(v1, v2, 2);
+
+    }
+
+    /**
+     * 减法 保留指定位小数
+     * 
+     * @param v1
+     * @param v2
+     * @param len
+     * @return
+     */
+    public static double subtract(double v1, double v2, int len) {
+        BigDecimal b1 = new BigDecimal(v1);
+        BigDecimal b2 = new BigDecimal(v2);
+        return b1.subtract(b2).setScale(len, BigDecimal.ROUND_HALF_UP).doubleValue();
+
+    }
+    /**差值绝对值
+     * @param v1
+     * @param v2
+     * @param len
+     * @return
+     */
+    public static double absoluteDiff(double v1, double v2, int len) {
+        BigDecimal b1 = new BigDecimal(v1);
+        BigDecimal b2 = new BigDecimal(v2);
+        if(v1>v2) {
+            return b1.subtract(b2).setScale(len, BigDecimal.ROUND_HALF_UP).doubleValue();
+        }else {
+            return b2.subtract(b1).setScale(len, BigDecimal.ROUND_HALF_UP).doubleValue();
+        }
+
+    }
+    /**
+     * 乘法 保留两位小数
+     * 
+     * @param v1
+     * @param v2
+     * @return
+     */
+    public static double multiply(double v1, double v2) {
+        return multiply(v1, v2, 2);
+
+    }
+
+    /**
+     * 乘法 保留指定位小数
+     * 
+     * @param v1
+     * @param v2
+     * @param len
+     * @return
+     */
+    public static double multiply(double v1, double v2, int len) {
+        BigDecimal b1 = new BigDecimal(v1);
+        BigDecimal b2 = new BigDecimal(v2);
+        return b1.multiply(b2).setScale(len, BigDecimal.ROUND_HALF_UP).doubleValue();
+
+    }
+    
+    
+    /**
+     * 除法 保留两位小数
+     * 
+     * @param v1
+     * @param v2
+     * @param len
+     * @return
+     */
+    public static double divide(double v1, double v2) {
+        return divide(v1, v2, 2);
+    }
+
+    /**
+     * 除法 保留指定位小数
+     * 
+     * @param v1
+     * @param v2
+     * @param len
+     * @return
+     */
+    public static double divide(double v1, double v2, int len) {
+        BigDecimal b1 = new BigDecimal(v1);
+        BigDecimal b2 = new BigDecimal(v2);
+        return b1.divide(b2, len, BigDecimal.ROUND_HALF_UP).doubleValue();
+    }
+
+    public static String divide2String(Double v1, Double v2, int len) {
+        if(v1==null||v2==null||v2==0) {
+            return "-";
+        }
+        BigDecimal b1 = new BigDecimal(v1);
+        BigDecimal b2 = new BigDecimal(v2);
+        return String.valueOf(b1.divide(b2, len, BigDecimal.ROUND_HALF_UP).doubleValue());
+    }
+}

src/main/java/cn/com/qmth/scancloud/tools/utils/SpringContextHolder.java

@@ -0,0 +1,74 @@
+package cn.com.qmth.scancloud.tools.utils;
+
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+import org.springframework.core.ResolvableType;
+import org.springframework.stereotype.Component;
+
+/**
+ * spring context holder.
+ *
+ */
+@Component
+public class SpringContextHolder implements ApplicationContextAware {
+
+	private static ApplicationContext ctx = null;
+
+	@Override
+	public void setApplicationContext(ApplicationContext ctx) {
+		SpringContextHolder.ctx = ctx;
+	}
+
+	public static ApplicationContext getApplicationContext() {
+		return ctx;
+	}
+
+	public static Object getBean(String name) {
+		return ctx.getBean(name);
+	}
+
+	public static <T> T getBean(String name, Class<T> requiredType) {
+		return ctx.getBean(name, requiredType);
+	}
+
+	public static <T> T getBean(Class<T> requiredType) {
+		return ctx.getBean(requiredType);
+	}
+
+	public static Object getBean(String name, Object... args) {
+		return ctx.getBean(name, args);
+	}
+
+	public static <T> T getBean(Class<T> requiredType, Object... args) {
+		return ctx.getBean(requiredType, args);
+	}
+
+	public static boolean containsBean(String name) {
+		return ctx.containsBean(name);
+	}
+
+	public static boolean isSingleton(String name) {
+		return ctx.isSingleton(name);
+	}
+
+	public static boolean isPrototype(String name) {
+		return ctx.isPrototype(name);
+	}
+
+	public static boolean isTypeMatch(String name, ResolvableType typeToMatch) {
+		return ctx.isTypeMatch(name, typeToMatch);
+	}
+
+	public static boolean isTypeMatch(String name, Class<?> typeToMatch) {
+		return ctx.isTypeMatch(name, typeToMatch);
+	}
+
+	public static Class<?> getType(String name) {
+		return ctx.getType(name);
+	}
+
+	public static String[] getAliases(String name) {
+		return ctx.getAliases(name);
+	}
+
+}

src/main/java/cn/com/qmth/scancloud/tools/utils/StatusException.java

@@ -10,5 +10,9 @@ public class StatusException extends RuntimeException {
     public StatusException(String message) {
         super(message);
     }
+    
+    public StatusException(String message,Throwable cause) {
+        super(message,cause);
+    }
 
 }

src/main/resources/application.properties

@@ -7,7 +7,8 @@ logging.level.cn.com.qmth.scancloud=info
 logging.pattern.console=%d{yyyy-MM-dd HH:mm:ss.SSS} | %clr(%-5level) | %msg%n
 logging.file.name=logs/debug.log
 scan.tool.task-type=OBJECTIVE_QUESTION_EXPORT
-scan.tool.thread.size=5
+scan.tool.thread.size=8
+scan.tool.batch.size=5000
 scan.tool.template.separator=,
 #############system params config###############