去重复制

examcloud-core-questions-api-provider/src/main/java/cn/com/qmth/examcloud/core/questions/api/controller/CopyDataController.java

@@ -1,18 +1,27 @@
 package cn.com.qmth.examcloud.core.questions.api.controller;
 
+import javax.servlet.http.HttpServletResponse;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
+import cn.com.qmth.examcloud.core.questions.service.CopyDataService;
+import cn.com.qmth.examcloud.web.redis.RedisClient;
 import cn.com.qmth.examcloud.web.support.ControllerSupport;
+import cn.com.qmth.examcloud.web.support.ServletUtil;
 
 @RestController
 @RequestMapping("${api_cqb}/copy")
 public class CopyDataController extends ControllerSupport {
-//	private static int cacheLockTimeout = 60 * 10;
-//	@Autowired
-//	private CopyDataService copyDataService;
-//	@Autowired
-//	private RedisClient redisClient;
+	private static int cacheLockTimeout = 60 * 10;
+	private static String withoutReduplicateLock = "$_COPY_QUESTION_DATA_NO_REDUPLICATE_LOCK";
+	@Autowired
+	private CopyDataService copyDataService;
+	@Autowired
+	private RedisClient redisClient;
 //
 //	//西交大数据复制
 //	@GetMapping("data")
@@ -34,5 +43,28 @@ public class CopyDataController extends ControllerSupport {
 //		}
 //		copyDataService.copyData(getAccessUser(), fromRootOrgId, toRootOrgId);
 //	}
+	
+	
+	//试题去重后复制
+	@GetMapping("noreduplicate")
+	public void copyNoReduplicate(HttpServletResponse response,@RequestParam String batch) {
+		Boolean lock = redisClient.setIfAbsent(withoutReduplicateLock, withoutReduplicateLock, cacheLockTimeout);
+		if (!lock) {
+			ServletUtil.returnJson("请求失败，正在处理数据", response);
+			return;
+		}
+		copyDataService.copyNoReduplicate(getAccessUser(), batch);
+		ServletUtil.returnJson("请求成功", response);
+	}
+	
+	@GetMapping("noreduplicate/status")
+	public void copyNoReduplicateStatus(HttpServletResponse response) {
+		String lock = redisClient.get(withoutReduplicateLock,String.class);
+		if (lock!=null) {
+			ServletUtil.returnJson("正在处理数据", response);
+		}else {
+			ServletUtil.returnJson("处理数据结束", response);
+		}
+	}
 
 }

examcloud-core-questions-base/src/main/java/cn/com/qmth/examcloud/core/questions/base/multithread/Consumer.java

@@ -66,6 +66,6 @@ public abstract class Consumer<T> extends Thread {
         return this.basket;
     }
     protected void processInfo() {
-        LOG.info(basket.getTaskName()+" 处理进度" + basket.getProgress());
+        LOG.warn(basket.getTaskName()+" 处理进度" + basket.getProgress());
     }
 }

examcloud-core-questions-service/src/main/java/cn/com/qmth/examcloud/core/questions/service/CopyDataService.java

@@ -6,5 +6,7 @@ public interface CopyDataService {
 
 	void copyData(User user, Long fromRootOrgId, Long toRootOrgId);
 
+	void copyNoReduplicate(User user,String batch);
+
 }
 

examcloud-core-questions-service/src/main/java/cn/com/qmth/examcloud/core/questions/service/bean/CopyDataNoReduplicateDto.java

@@ -0,0 +1,36 @@
+package cn.com.qmth.examcloud.core.questions.service.bean;
+
+import cn.com.qmth.examcloud.api.commons.security.bean.User;
+import cn.com.qmth.examcloud.core.basic.api.bean.CourseBean;
+
+public class CopyDataNoReduplicateDto {
+	private CourseBean course;
+	private User user;
+	private String batch;
+	public CourseBean getCourse() {
+		return course;
+	}
+	public void setCourse(CourseBean course) {
+		this.course = course;
+	}
+	public User getUser() {
+		return user;
+	}
+	public void setUser(User user) {
+		this.user = user;
+	}
+	public String getBatch() {
+		return batch;
+	}
+	public void setBatch(String batch) {
+		this.batch = batch;
+	}
+	public CopyDataNoReduplicateDto(CourseBean course, User user, String batch) {
+		super();
+		this.course = course;
+		this.user = user;
+		this.batch = batch;
+	}
+
+
+}

examcloud-core-questions-service/src/main/java/cn/com/qmth/examcloud/core/questions/service/consumer/CopyDataNoReduplicateConsumer.java

@@ -0,0 +1,436 @@
+package cn.com.qmth.examcloud.core.questions.service.consumer;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.bson.types.ObjectId;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Scope;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import cn.com.qmth.examcloud.api.commons.security.bean.User;
+import cn.com.qmth.examcloud.commons.exception.StatusException;
+import cn.com.qmth.examcloud.core.basic.api.bean.CourseBean;
+import cn.com.qmth.examcloud.core.questions.base.CommonUtils;
+import cn.com.qmth.examcloud.core.questions.base.IoUtils;
+import cn.com.qmth.examcloud.core.questions.base.StringSimilarityUtils;
+import cn.com.qmth.examcloud.core.questions.base.enums.PaperStatus;
+import cn.com.qmth.examcloud.core.questions.base.enums.PaperType;
+import cn.com.qmth.examcloud.core.questions.base.multithread.Consumer;
+import cn.com.qmth.examcloud.core.questions.base.question.enums.QuesStructType;
+import cn.com.qmth.examcloud.core.questions.dao.PaperDetailRepo;
+import cn.com.qmth.examcloud.core.questions.dao.PaperDetailUnitRepo;
+import cn.com.qmth.examcloud.core.questions.dao.PaperRepo;
+import cn.com.qmth.examcloud.core.questions.dao.QuesRepo;
+import cn.com.qmth.examcloud.core.questions.dao.QuestionAudioRepo;
+import cn.com.qmth.examcloud.core.questions.dao.entity.Course;
+import cn.com.qmth.examcloud.core.questions.dao.entity.Paper;
+import cn.com.qmth.examcloud.core.questions.dao.entity.PaperDetail;
+import cn.com.qmth.examcloud.core.questions.dao.entity.PaperDetailUnit;
+import cn.com.qmth.examcloud.core.questions.dao.entity.QuesOption;
+import cn.com.qmth.examcloud.core.questions.dao.entity.Question;
+import cn.com.qmth.examcloud.core.questions.dao.entity.QuestionAudio;
+import cn.com.qmth.examcloud.core.questions.service.QuestionAudioService;
+import cn.com.qmth.examcloud.core.questions.service.UpYunService;
+import cn.com.qmth.examcloud.core.questions.service.bean.CopyDataNoReduplicateDto;
+import cn.com.qmth.examcloud.core.questions.service.config.SysProperty;
+import cn.com.qmth.examcloud.support.filestorage.FileStorageUtil;
+import cn.com.qmth.examcloud.web.filestorage.FileStoragePathEnvInfo;
+import cn.com.qmth.examcloud.web.filestorage.YunPathInfo;
+import cn.com.qmth.examcloud.web.redis.RedisClient;
+
+@Scope("prototype")
+@Service
+public class CopyDataNoReduplicateConsumer extends Consumer<CopyDataNoReduplicateDto> {
+	private static int cacheLockTimeout = 60 * 10;
+	private static String withoutReduplicateLock = "$_COPY_QUESTION_DATA_NO_REDUPLICATE_LOCK";
+	@Autowired
+	private RedisClient redisClient;
+	@Autowired
+	private PaperRepo paperRepo;
+	@Autowired
+	private QuestionAudioRepo questionAudioRepo;
+	@Autowired
+	private SysProperty sysProperty;
+
+	@Autowired
+	private PaperDetailRepo paperDetailRepo;
+
+	@Autowired
+	private PaperDetailUnitRepo paperDetailUnitRepo;
+
+	@Autowired
+	private QuestionAudioService questionAudioService;
+
+	@Autowired
+	private QuesRepo quesRepo;
+
+	@Autowired
+	private MongoTemplate mongoTemplate;
+
+	@Transactional
+	@Override
+	public int consume(CopyDataNoReduplicateDto dto) {
+		List<Question> qs = findQuestion(dto.getUser().getRootOrgId(), dto.getCourse().getCode());
+		if (CollectionUtils.isEmpty(qs)) {
+			return 1;
+		}
+		Map<QuesStructType, List<Question>> map = new HashMap<>();
+		for (Question q : qs) {
+			List<Question> tem = map.get(q.getQuestionType());
+			if (tem == null) {
+				tem = new ArrayList<>();
+				map.put(q.getQuestionType(),tem);
+			}
+			tem.add(q);
+		}
+		for (QuesStructType qt : map.keySet()) {
+			removeReduplicate(qt, map, dto);
+		}
+		return 1;
+	}
+
+	private void removeReduplicate(QuesStructType qt, Map<QuesStructType, List<Question>> map,
+			CopyDataNoReduplicateDto dto) {
+		List<Question> qs = checkQuestion(map.get(qt));
+		createPapers(qs, qt, dto);
+	}
+
+	private void createPapers(List<Question> qs, QuesStructType qt, CopyDataNoReduplicateDto dto) {
+		if (qs == null || qs.size() == 0) {
+			return;
+		}
+		if (qs.size() <= 200) {
+			createPaper(qs, qt, dto, 1);
+			redisClient.expire(withoutReduplicateLock, cacheLockTimeout);
+		} else {
+			int size = qs.size();
+			int len = 200;
+			int count = (size + len - 1) / len;
+
+			for (int i = 0; i < count; i++) {
+				List<Question> subList = qs.subList(i * len, ((i + 1) * len > size ? size : len * (i + 1)));
+				createPaper(subList, qt, dto, i + 1);
+				redisClient.expire(withoutReduplicateLock, cacheLockTimeout);
+			}
+		}
+
+	}
+
+	private void createPaper(List<Question> qs, QuesStructType qt, CopyDataNoReduplicateDto dto, int indx) {
+		if (qs.size() == 0) {
+			return;
+		}
+		Course course = of(dto.getCourse());
+		Paper paper = new Paper();
+		if(QuesStructType.NESTED_ANSWER_QUESTION.equals(qt)) {
+			paper.setName("(" + dto.getBatch() + ")" + qt.getName() + "_" + indx);
+		}else {
+			paper.setName("(" + dto.getBatch() + ")" + qt.getName() + "题_" + indx);
+		}
+		paper.setTitle(paper.getName());
+		paper.setTotalScore((double) qs.size());
+		paper.setCreator(dto.getUser().getDisplayName());
+		paper.setPaperDetailCount(1);
+		paper.setUnitCount(qs.size());
+		paper.setLastModifyName(dto.getUser().getDisplayName());
+		paper.setPaperStatus(PaperStatus.DRAFT);
+		paper.setPaperType(PaperType.IMPORT);
+		paper.setCourseNo(dto.getCourse().getCode());
+		paper.setCourseName(dto.getCourse().getName());
+		paper.setCourse(course);
+		paper.setOrgId(dto.getUser().getRootOrgId().toString());
+		paper.setDifficultyDegree(0.5);
+		paper.setAuditStatus(false);
+		paper.setCreationBy(dto.getUser().getUserId());
+		paperRepo.save(paper);
+
+		PaperDetail detail = new PaperDetail();
+		detail.setPaper(paper);
+		if(QuesStructType.NESTED_ANSWER_QUESTION.equals(qt)) {
+			detail.setName(qt.getName());
+		}else {
+			detail.setName(qt.getName() + "题");
+		}
+		detail.setNumber(1);
+		detail.setScore((double) qs.size());
+		detail.setUnitCount(qs.size());
+		detail.setCreator(dto.getUser().getDisplayName());
+		detail.setCreateTime(CommonUtils.getCurDateTime());
+		detail.setCreationBy(dto.getUser().getUserId());
+		paperDetailRepo.save(detail);
+		createUnit(paper, detail, qs, qt, dto);
+	}
+
+	private void createUnit(Paper paper, PaperDetail detail, List<Question> qs, QuesStructType qt,
+			CopyDataNoReduplicateDto dto) {
+		List<PaperDetailUnit> units = new ArrayList<>();
+		int index = 0;
+		List<QuestionAudio> audios = new ArrayList<>();
+		for (Question q : qs) {
+			index++;
+			String oldId = q.getId();
+			q.setId(new ObjectId().toString());
+			q.setCreateTime(CommonUtils.getCurDateTime());
+			q.setUpdateBy(null);
+			q.setUpdateDate(null);
+			q.setUpdateTime(null);
+			q.setScore(1.0);
+			q.setCreationBy(dto.getUser().getUserId());
+			// 复制试题的音频
+			List<QuestionAudio> audioList = questionAudioService.findQuestionAudiosByQuestionId(oldId);
+			if (audioList != null && audioList.size() > 0) {
+				for (QuestionAudio oldAudio : audioList) {
+					QuestionAudio audio = this.copyQuestionAudio(oldAudio, q, dto.getUser());
+					q.addAudio(audio);
+				}
+				audios.addAll(q.getAudioList());
+			}
+			//设置音频新路径
+			dealQuestionAudios(q);
+			//替换试题音频id
+			updateQuestionBody(q, q.getAudioList());
+			PaperDetailUnit unit = new PaperDetailUnit();
+			units.add(unit);
+			unit.setPaper(paper);
+			unit.setNumber(index);
+			unit.setScore(1.0);
+			unit.setPaperDetail(detail);
+			unit.setQuestionType(q.getQuestionType());
+			unit.setQuestion(q);
+			unit.setCreator(dto.getUser().getDisplayName());
+			unit.setCreateTime(CommonUtils.getCurDateTime());
+			unit.setCreationBy(dto.getUser().getUserId());
+			unit.setPaperType(PaperType.IMPORT);
+		}
+		quesRepo.saveAll(qs);
+		if (audios.size() > 0) {
+			questionAudioRepo.saveAll(audios);
+		}
+		paperDetailUnitRepo.saveAll(units);
+	}
+
+	private Course of(CourseBean c) {
+		Course ret = new Course();
+		ret.setCode(c.getCode());
+		ret.setEnable(c.getEnable() == null ? null : c.getEnable().toString());
+		ret.setId(c.getId().toString());
+		ret.setLevel(c.getLevel());
+		ret.setName(c.getName());
+		ret.setOrgId(c.getRootOrgId().toString());
+		return ret;
+	}
+
+	private List<Question> checkQuestion(List<Question> dtos) {
+		List<Question> ret = new ArrayList<>();
+		Set<String> checkIds = new HashSet<>();
+		Map<String, String> extractTextMap = new HashMap<>();
+		for (Question dto : dtos) {
+			String text = getExtractText(dto);
+			if (StringUtils.isBlank(text)) {
+				checkIds.add(dto.getId());
+				ret.add(dto);
+			} else {
+				extractTextMap.put(dto.getId(), text);
+			}
+		}
+		int length = dtos.size();
+		for (int i = 0; i < length; i++) {
+			Question dto = dtos.get(i);
+			if (checkIds.contains(dto.getId())) {
+				continue;
+			}
+			ret.add(dto);
+			String quesText1 = extractTextMap.get(dto.getId());
+			for (int j = i + 1; j < length; j++) {
+				Question subdto = dtos.get(j);
+				if (checkIds.contains(subdto.getId())) {
+					continue;
+				}
+				String quesText2 = extractTextMap.get(subdto.getId());
+				double similarity = StringSimilarityUtils.getSimilarityWithCosinesBySeg(quesText1, quesText2);
+				if (similarity > 0.94) {
+					checkIds.add(dto.getId());
+					checkIds.add(subdto.getId());
+				}
+			}
+		}
+		return ret;
+	}
+
+	private String getExtractText(Question question) {
+		StringBuilder quesText = new StringBuilder();
+		if (question == null) {
+			return quesText.toString();
+		}
+
+		if (!StringUtils.isEmpty(question.getQuesBody())) {
+			quesText.append(getTextInHtml(question.getQuesBody()));
+		}
+
+		List<QuesOption> quesOptionList = question.getQuesOptions();
+		if (quesOptionList != null && quesOptionList.size() > 0) {
+			for (QuesOption quesOption : quesOptionList) {
+				if (!StringUtils.isEmpty(quesOption.getOptionBody())) {
+					quesText.append(getTextInHtml(quesOption.getOptionBody()));
+				}
+			}
+		}
+
+		return quesText.toString();
+	}
+
+	private String getTextInHtml(String htmlStr) {
+		htmlStr = htmlStr.replaceAll("\\&[a-zA-Z]{1,10};", "").trim();
+		if (!htmlStr.startsWith("<p>")) {
+			return htmlStr;
+		}
+
+		try {
+			org.jsoup.nodes.Document doc = Jsoup.parse(htmlStr);
+			StringBuilder textStr = new StringBuilder();
+			Elements links = doc.select("p").removeAttr("img");
+
+			for (Element link : links) {
+				textStr.append(link.text().trim());
+			}
+
+			return textStr.toString();
+		} catch (Exception e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+	private List<Question> findQuestion(Long rootOrgId, String courseCode) {
+		Query query = new Query();
+		query.addCriteria(Criteria.where("orgId").is(rootOrgId.toString()));
+		query.addCriteria(Criteria.where("course.code").is(courseCode));
+		List<Question> ret = this.mongoTemplate.find(query, Question.class, "question");
+		return ret;
+	}
+
+	/**
+	 * 更新试题题干或选项中的音频存储ID
+	 */
+	private void updateQuestionBody(Question question, List<QuestionAudio> audioList) {
+		if(CollectionUtils.isEmpty(audioList)) {
+			return;
+		}
+		Map<String, String> audioMaps = new HashMap<>();
+		for (QuestionAudio audio : audioList) {
+			audioMaps.put(audio.getFileName(), audio.getId());
+		}
+
+		// 在题干html片段中替换音频存储ID
+		String body = question.getQuesBody();
+		List<String> ids = CommonUtils.getTagANames(body);
+		List<String> names = CommonUtils.getTagANames2(body);
+		Map<String, String> oldBodyMap = new HashMap<>();
+		for (int i = 0; i < ids.size(); i++) {
+			oldBodyMap.put(names.get(i), ids.get(i));
+		}
+		for (String key : oldBodyMap.keySet()) {
+			body = body.replace(oldBodyMap.get(key), audioMaps.get(key));
+		}
+		question.setQuesBody(body);
+
+		if (question.getQuesOptions() == null || question.getQuesOptions().size() == 0) {
+			return;
+		}
+
+		// 替换选项中的音频存储ID
+		for (QuesOption option : question.getQuesOptions()) {
+			String newOptionBody = option.getOptionBody();
+			List<String> optionIds = CommonUtils.getTagANames(newOptionBody);
+			List<String> optionNames = CommonUtils.getTagANames2(newOptionBody);
+
+			Map<String, String> oldOptionMap = new HashMap<>();
+			for (int i = 0; i < optionIds.size(); i++) {
+				oldOptionMap.put(optionNames.get(i), optionIds.get(i));
+			}
+			for (String key : oldOptionMap.keySet()) {
+				newOptionBody = newOptionBody.replace(oldOptionMap.get(key), audioMaps.get(key));
+			}
+			option.setOptionBody(newOptionBody);
+		}
+	}
+
+	/**
+	 * 处理试题的音频(下载上传到云存储)
+	 */
+	private void dealQuestionAudios(Question question) {
+		String copyAudioPath = UpYunService.TEMP_FILE_EXP + File.separator + randomUUID();
+		File copyAudioDir = new File(copyAudioPath);
+		if (!copyAudioDir.exists()) {
+			copyAudioDir.mkdirs();
+		}
+		List<QuestionAudio> audioList = question.getAudioList();
+		if (audioList == null || audioList.size() == 0) {
+			return;
+		}
+		for (QuestionAudio audio : audioList) {
+			// 定义文件下载名称，并下载音频文件
+			String newAudioFileName = randomUUID() + "_" + audio.getFileName();
+			File audioFile = new File(copyAudioPath + File.separator + newAudioFileName);
+
+//                    upYun.readFile(audio.getFileUrl(), audioFile);
+			// 通用存储
+			FileStorageUtil.saveUrlAs(FileStorageUtil.realPath(audio.getFileUrl()), audioFile);
+
+			// 重新上传新的音频文件
+			String newPath = sysProperty.getRadioUploadPath() + newAudioFileName;
+			try {
+//                        upYun.writeFile(newPath, audioFile, true);
+				// 通用存储
+				FileStoragePathEnvInfo env = new FileStoragePathEnvInfo();
+				env.setRelativePath(newPath);
+				YunPathInfo pi = FileStorageUtil.saveFile("audioFile", env, audioFile, null);
+				audio.setFileUrl(pi.getRelativePath());// 设置新路径
+			} catch (Exception e) {
+				throw new StatusException("上传音频文件失败！");
+			} finally {
+				IoUtils.removeFile(audioFile);
+			}
+		}
+
+		IoUtils.removeFile(copyAudioDir);
+	}
+
+	/**
+	 * 复制试题的音频
+	 */
+	private QuestionAudio copyQuestionAudio(QuestionAudio oldAudio, Question q, User user) {
+		QuestionAudio audio = new QuestionAudio();
+		audio.setId(new ObjectId().toString());
+		audio.setQuestionId(q.getId());
+		audio.setFileUrl(oldAudio.getFileUrl());
+		audio.setFileName(oldAudio.getFileName());
+		audio.setFileSuffixes(oldAudio.getFileSuffixes());
+		audio.setCreateTime(new Date());
+		audio.setCreateUser(user.getDisplayName());
+		audio.setCreationBy(user.getUserId());
+		audio.setUpdateBy(null);
+		audio.setUpdateDate(null);
+		return audio;
+	}
+
+	private String randomUUID() {
+		return UUID.randomUUID().toString().replaceAll("-", "");
+	}
+}

examcloud-core-questions-service/src/main/java/cn/com/qmth/examcloud/core/questions/service/impl/CopyDataServiceImpl.java

@@ -9,6 +9,7 @@ import org.apache.commons.collections4.CollectionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
@@ -28,13 +29,16 @@ import cn.com.qmth.examcloud.core.questions.dao.entity.QuesTypeName;
 import cn.com.qmth.examcloud.core.questions.service.CopyDataService;
 import cn.com.qmth.examcloud.core.questions.service.PaperService;
 import cn.com.qmth.examcloud.core.questions.service.bean.CopyDataDto;
+import cn.com.qmth.examcloud.core.questions.service.bean.CopyDataNoReduplicateDto;
+import cn.com.qmth.examcloud.core.questions.service.producer.CopyDataNoReduplicateProducer;
 import cn.com.qmth.examcloud.core.questions.service.producer.CopyDataProducer;
 import cn.com.qmth.examcloud.web.redis.RedisClient;
 
 @Service("copyDataService")
 public class CopyDataServiceImpl implements CopyDataService {
 	private static final Logger log = LoggerFactory.getLogger(CopyDataService.class);
-	private static int cacheLockTimeout=60*10;
+	private static int cacheLockTimeout = 60 * 10;
+	private static String withoutReduplicateLock = "$_COPY_QUESTION_DATA_NO_REDUPLICATE_LOCK";
 	@Autowired
 	private CourseCloudService courseCloudService;
 
@@ -43,6 +47,9 @@ public class CopyDataServiceImpl implements CopyDataService {
 
 	@Autowired
 	private CopyDataProducer copyDataProducer;
+	
+	@Autowired
+	private CopyDataNoReduplicateProducer copyDataNoReduplicateProducer;
 
 	@Autowired
 	private PaperDetailRepo paperDetailRepo;
@@ -63,13 +70,67 @@ public class CopyDataServiceImpl implements CopyDataService {
 	@Autowired
 	private RedisClient redisClient;
 
+	@Async
+	@Override
+	@Transactional
+	public void copyNoReduplicate(User user, String batch) {
+		Long rootOrgId = user.getRootOrgId();
+		boolean sucss = true;
+		long s1 = System.currentTimeMillis();
+		try {
+			log.warn("数据复制开始 | rootOrgId=" + rootOrgId);
+			disposeCopyNoReduplicate(user, batch);
+		} catch (Exception e) {
+			sucss = false;
+			throw e;
+		} finally {
+			long s2 = System.currentTimeMillis();
+			if (sucss) {
+				log.warn("数据复制结束 | 成功 |  " + ((s2 - s1) / 1000) + "秒 |  rootOrgId=" + rootOrgId);
+			} else {
+				log.warn("数据复制结束 | 失败 |  " + ((s2 - s1) / 1000) + "秒 |  rootOrgId=" + rootOrgId);
+			}
+			redisClient.delete(withoutReduplicateLock);
+		}
+	}
+
+	private void disposeCopyNoReduplicate(User user, String batch) {
+		Long rootOrgId=user.getRootOrgId();
+		redisClient.expire(withoutReduplicateLock, cacheLockTimeout);
+		List<CourseBean> courses = new ArrayList<>();
+		GetCourseByOrgReq courseReq = new GetCourseByOrgReq();
+		courseReq.setRootOrgId(rootOrgId);
+		GetCourseByOrgResp res = courseCloudService.getCourseByOrg(courseReq);
+		if (CollectionUtils.isEmpty(res.getCourseList())) {
+			log.warn("没有课程数据");
+		}
+		for (CourseBean c : res.getCourseList()) {
+			if (c.getEnable()) {
+				courses.add(c);
+			}
+		}
+		if (courses.size() == 0) {
+			log.warn("没有课程数据");
+		}
+		redisClient.expire(withoutReduplicateLock, cacheLockTimeout);
+		List<CopyDataNoReduplicateDto> dtos = new ArrayList<>();
+		for (CourseBean c : courses) {
+			dtos.add(new CopyDataNoReduplicateDto(c, user,batch));
+		}
+		Map<String, Object> param = new HashMap<>();
+		param.put("dtos", dtos);
+		copyDataNoReduplicateProducer.startDispose(4, param, dtos.size());
+		String countInfo = "成功科目数:" + copyDataNoReduplicateProducer.getProcess() + " 科目总数:" + copyDataNoReduplicateProducer.getTotal();
+		log.warn(countInfo);
+	}
+
 	@Override
 	@Transactional
 	public void copyData(User user, Long fromRootOrgId, Long toRootOrgId) {
 		boolean sucss = true;
 		long s1 = System.currentTimeMillis();
 		try {
-			log.info("数据复制开始 | fromRootOrgId=" + fromRootOrgId + " | toRootOrgId=" + toRootOrgId);
+			log.warn("数据复制开始 | fromRootOrgId=" + fromRootOrgId + " | toRootOrgId=" + toRootOrgId);
 			dispose(user, fromRootOrgId, toRootOrgId);
 		} catch (Exception e) {
 			sucss = false;
@@ -77,10 +138,10 @@ public class CopyDataServiceImpl implements CopyDataService {
 		} finally {
 			long s2 = System.currentTimeMillis();
 			if (sucss) {
-				log.info("数据复制结束 | 成功 |  " + ((s2 - s1) / 1000) + "秒 |  fromRootOrgId=" + fromRootOrgId
+				log.warn("数据复制结束 | 成功 |  " + ((s2 - s1) / 1000) + "秒 |  fromRootOrgId=" + fromRootOrgId
 						+ " | toRootOrgId=" + toRootOrgId);
 			} else {
-				log.info("数据复制结束 | 失败 |  " + ((s2 - s1) / 1000) + "秒 |  fromRootOrgId=" + fromRootOrgId
+				log.warn("数据复制结束 | 失败 |  " + ((s2 - s1) / 1000) + "秒 |  fromRootOrgId=" + fromRootOrgId
 						+ " | toRootOrgId=" + toRootOrgId);
 			}
 			String cacheLock = "$_COPY_QUESTION_DATA_LOCK";
@@ -106,16 +167,16 @@ public class CopyDataServiceImpl implements CopyDataService {
 		List<String> paperIds = paperService.findPaperId(fromRootOrgId);
 		if (CollectionUtils.isNotEmpty(paperIds)) {
 			List<CopyDataDto> dtos = new ArrayList<>();
-			for(String paperId:paperIds) {
+			for (String paperId : paperIds) {
 				dtos.add(new CopyDataDto(courses, paperId, toRootOrgId, user));
 			}
 			Map<String, Object> param = new HashMap<>();
 			param.put("dtos", dtos);
 			copyDataProducer.startDispose(8, param, dtos.size());
 			String countInfo = "成功数:" + copyDataProducer.getProcess() + " 总数:" + copyDataProducer.getTotal();
-			log.info(countInfo);
-		}else {
-			log.info("没有试卷数据需要处理");
+			log.warn(countInfo);
+		} else {
+			log.warn("没有试卷数据需要处理");
 		}
 	}
 

examcloud-core-questions-service/src/main/java/cn/com/qmth/examcloud/core/questions/service/producer/CopyDataNoReduplicateProducer.java

@@ -0,0 +1,36 @@
+package cn.com.qmth.examcloud.core.questions.service.producer;
+
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import cn.com.qmth.examcloud.core.questions.base.multithread.Producer;
+import cn.com.qmth.examcloud.core.questions.service.bean.CopyDataNoReduplicateDto;
+import cn.com.qmth.examcloud.core.questions.service.consumer.CopyDataNoReduplicateConsumer;
+import cn.com.qmth.examcloud.web.redis.RedisClient;
+
+@Service
+public class CopyDataNoReduplicateProducer extends Producer<CopyDataNoReduplicateDto, CopyDataNoReduplicateConsumer> {
+	private static int cacheLockTimeout=60*10;
+	
+	@Autowired
+	private RedisClient redisClient;
+    @SuppressWarnings("unchecked")
+    @Override
+    protected CopyDataNoReduplicateDto findData(Map<String, Object> param, int index) {
+        List<CopyDataNoReduplicateDto> ret=(List<CopyDataNoReduplicateDto>)param.get("dtos");
+        if(index>=ret.size()) {
+            return null;
+        }
+        redisClient.expire("$_COPY_QUESTION_DATA_NO_REDUPLICATE_LOCK", cacheLockTimeout);
+        return ret.get(index);
+    }
+
+    @Override
+    protected String getTaskName() {
+        return "试题去重复制";
+    }
+
+}