查重

pom.xml

@@ -1,101 +1,108 @@
 <?xml version="1.0"?>
-<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
-         xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
-    <modelVersion>4.0.0</modelVersion>
-    <artifactId>examcloud-oe-tool</artifactId>
-    <version>v4.1.3-RELEASE</version>
-    <packaging>jar</packaging>
+<project
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
+	xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+	<modelVersion>4.0.0</modelVersion>
+	<artifactId>examcloud-oe-tool</artifactId>
+	<version>v4.1.3-RELEASE</version>
+	<packaging>jar</packaging>
 
-    <parent>
-        <groupId>cn.com.qmth.examcloud</groupId>
-        <artifactId>examcloud-parent</artifactId>
-        <version>v4.1.4-RELEASE</version>
-    </parent>
+	<parent>
+		<groupId>cn.com.qmth.examcloud</groupId>
+		<artifactId>examcloud-parent</artifactId>
+		<version>v4.1.4-SNAPSHOT</version>
+	</parent>
 
-    <properties>
-        <!-- 云平台版本 -->
-        <examcloud.version>v4.1.4-RELEASE</examcloud.version>
-    </properties>
+	<properties>
+		<!-- 云平台版本 -->
+		<examcloud.version>v4.1.4-SNAPSHOT</examcloud.version>
+	</properties>
 
-    <dependencies>
-        <dependency>
-            <groupId>cn.com.qmth.examcloud</groupId>
-            <artifactId>examcloud-web</artifactId>
-            <version>${examcloud.version}</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.springframework.cloud</groupId>
-                    <artifactId>spring-cloud-starter-netflix-eureka-client</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-        <dependency>
-            <groupId>cn.com.qmth.examcloud</groupId>
-            <artifactId>examcloud-support</artifactId>
-            <version>${examcloud.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.projectlombok</groupId>
-            <artifactId>lombok</artifactId>
-            <version>1.18.20</version>
-        </dependency>
+	<dependencies>
+		<dependency>
+			<groupId>cn.com.qmth.examcloud</groupId>
+			<artifactId>examcloud-web</artifactId>
+			<version>${examcloud.version}</version>
+			<exclusions>
+				<exclusion>
+					<groupId>org.springframework.cloud</groupId>
+					<artifactId>spring-cloud-starter-netflix-eureka-client</artifactId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+		<dependency>
+			<groupId>cn.com.qmth.examcloud</groupId>
+			<artifactId>examcloud-support</artifactId>
+			<version>${examcloud.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.ansj</groupId>
+			<artifactId>ansj_seg</artifactId>
+			<version>5.1.6</version>
+		</dependency>
+		<dependency>
+			<groupId>org.projectlombok</groupId>
+			<artifactId>lombok</artifactId>
+			<version>1.18.20</version>
+		</dependency>
 
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-data-mongodb</artifactId>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.springframework.boot</groupId>
-                    <artifactId>spring-boot-starter-logging</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-    </dependencies>
+		<dependency>
+			<groupId>org.springframework.boot</groupId>
+			<artifactId>spring-boot-starter-data-mongodb</artifactId>
+			<exclusions>
+				<exclusion>
+					<groupId>org.springframework.boot</groupId>
+					<artifactId>spring-boot-starter-logging</artifactId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+	</dependencies>
 
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-jar-plugin</artifactId>
-                <configuration>
-                    <archive>
-                        <manifest>
-                            <mainClass>cn.com.qmth.dp.examcloud.oe.OeToolApplication</mainClass>
-                            <addClasspath>true</addClasspath>
-                            <classpathPrefix>./</classpathPrefix>
-                        </manifest>
-                        <manifestEntries>
-                            <Class-Path>../config/</Class-Path>
-                        </manifestEntries>
-                    </archive>
-                    <excludes>
-                        <exclude>*.sql</exclude>
-                        <exclude>*.properties</exclude>
-                        <exclude>*.xml</exclude>
-                        <exclude>classpath.location</exclude>
-                    </excludes>
-                </configuration>
-            </plugin>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <configuration>
-                    <finalName>examcloud-oe-tool</finalName>
-                    <descriptors>
-                        <descriptor>assembly.xml</descriptor>
-                    </descriptors>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>make-assembly</id>
-                        <phase>install</phase>
-                        <goals>
-                            <goal>assembly</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-jar-plugin</artifactId>
+				<configuration>
+					<archive>
+						<manifest>
+							<mainClass>cn.com.qmth.dp.examcloud.oe.OeToolApplication</mainClass>
+							<addClasspath>true</addClasspath>
+							<classpathPrefix>./</classpathPrefix>
+						</manifest>
+						<manifestEntries>
+							<Class-Path>../config/</Class-Path>
+						</manifestEntries>
+					</archive>
+					<excludes>
+						<exclude>*.sql</exclude>
+						<exclude>*.properties</exclude>
+						<exclude>*.xml</exclude>
+						<exclude>classpath.location</exclude>
+					</excludes>
+				</configuration>
+			</plugin>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-assembly-plugin</artifactId>
+				<configuration>
+					<finalName>examcloud-oe-tool</finalName>
+					<descriptors>
+						<descriptor>assembly.xml</descriptor>
+					</descriptors>
+				</configuration>
+				<executions>
+					<execution>
+						<id>make-assembly</id>
+						<phase>install</phase>
+						<goals>
+							<goal>assembly</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
+		</plugins>
+	</build>
 
 </project>

src/main/java/cn/com/qmth/dp/examcloud/oe/Task.java

@@ -1,23 +1,9 @@
 package cn.com.qmth.dp.examcloud.oe;
 
-import cn.com.qmth.dp.examcloud.oe.modules.cut_exam_offline_data.CutExamOfflineDataService;
-import cn.com.qmth.dp.examcloud.oe.modules.exam_record_data.ExamRecordDataTool;
-import cn.com.qmth.dp.examcloud.oe.modules.export_exam_student_score.ExportExamStudentScore;
-import cn.com.qmth.dp.examcloud.oe.modules.fixExamStudentId.FixExamStudentId;
-import cn.com.qmth.dp.examcloud.oe.modules.get_student_answer_detail.GetStduentAnswerDetailService;
-import cn.com.qmth.dp.examcloud.oe.modules.get_student_one_question_answer.GetStduentOneAnswerService;
-import cn.com.qmth.dp.examcloud.oe.modules.import_paper_dzkd.ImportPaperDzkdService;
-import cn.com.qmth.dp.examcloud.oe.modules.init_user_data_rule.InitUserDataRule;
-import cn.com.qmth.dp.examcloud.oe.modules.marking_item_change.MarkingItemChangeService;
-import cn.com.qmth.dp.examcloud.oe.modules.update_correct_answer.FixCorrectAnswerAndResetScoreService;
-import cn.com.qmth.examcloud.commons.util.JsonUtil;
-import cn.com.qmth.examcloud.web.support.SpringContextHolder;
-import cn.com.qmth.examcloud.web.upyun.UpYunPathInfo;
-import cn.com.qmth.examcloud.web.upyun.UpyunPathEnvironmentInfo;
-import cn.com.qmth.examcloud.web.upyun.UpyunService;
-import com.mongodb.client.FindIterable;
-import com.mongodb.client.MongoCollection;
-import com.mongodb.client.MongoCursor;
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+
 import org.apache.commons.lang3.RandomUtils;
 import org.bson.Document;
 import org.slf4j.Logger;
@@ -26,9 +12,16 @@ import org.springframework.data.mongodb.core.MongoTemplate;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Component;
 
-import java.io.File;
-import java.util.List;
-import java.util.Map;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoCursor;
+
+import cn.com.qmth.dp.examcloud.oe.modules.findreduplicatequestion.ExportReduplicateQuestionService;
+import cn.com.qmth.examcloud.commons.util.JsonUtil;
+import cn.com.qmth.examcloud.web.support.SpringContextHolder;
+import cn.com.qmth.examcloud.web.upyun.UpYunPathInfo;
+import cn.com.qmth.examcloud.web.upyun.UpyunPathEnvironmentInfo;
+import cn.com.qmth.examcloud.web.upyun.UpyunService;
 
 /**
  * 任务
@@ -58,7 +51,7 @@ public class Task {
             // SpringContextHolder.getBean(InitUserDataRule.class).start();
             // SpringContextHolder.getBean(FixExamStudentId.class).start(2843L, null);
             // SpringContextHolder.getBean(MarkingItemChangeService.class).start();
-            // SpringContextHolder.getBean(ExamRecordDataTool.class).start();
+             SpringContextHolder.getBean(ExportReduplicateQuestionService.class).start();
 
         } catch (Exception e) {
             log.error("unexpected", e);

src/main/java/cn/com/qmth/dp/examcloud/oe/modules/findreduplicatequestion/DetailDto.java

@@ -0,0 +1,29 @@
+package cn.com.qmth.dp.examcloud.oe.modules.findreduplicatequestion;
+
+import javax.persistence.Id;
+
+public class DetailDto {
+	@Id
+    private String id;
+	private String name;
+	private Integer number;
+	public String getId() {
+		return id;
+	}
+	public void setId(String id) {
+		this.id = id;
+	}
+	public String getName() {
+		return name;
+	}
+	public void setName(String name) {
+		this.name = name;
+	}
+	public Integer getNumber() {
+		return number;
+	}
+	public void setNumber(Integer number) {
+		this.number = number;
+	}
+	
+}

src/main/java/cn/com/qmth/dp/examcloud/oe/modules/findreduplicatequestion/ExportQuesReduplicateConsumer.java

@@ -0,0 +1,227 @@
+package cn.com.qmth.dp.examcloud.oe.modules.findreduplicatequestion;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.bson.types.ObjectId;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Scope;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.stereotype.Service;
+
+import cn.com.qmth.dp.examcloud.oe.entity.question.Course;
+import cn.com.qmth.dp.examcloud.oe.enums.question.PaperType;
+import cn.com.qmth.dp.examcloud.oe.enums.question.QuesStructType;
+import cn.com.qmth.dp.examcloud.oe.multithread.Consumer;
+import cn.com.qmth.dp.examcloud.oe.util.PaperUtil;
+import cn.com.qmth.dp.examcloud.oe.util.StringSimilarityUtils;
+
+@Service
+@Scope("prototype")
+public class ExportQuesReduplicateConsumer extends Consumer<Course> {
+
+	@Autowired
+	private MongoTemplate mongoTemplate;
+
+	@Override
+	public void consume(Map<String, Object> param, Course c) {
+		String rootOrgId = (String) param.get("rootOrgId");
+//		String paperName = (String) param.get("paperName");
+//		List<PaperDto> papers = findPaper(rootOrgId, c.getCode(), paperName);
+//		if (CollectionUtils.isEmpty(papers)) {
+//			return;
+//		}
+//		List<ObjectId> paperIds = papers.stream().map(e -> new ObjectId(e.getId())).collect(Collectors.toList());
+//		List<UnitDto> units = findUnit(paperIds);
+//		if (CollectionUtils.isEmpty(units)) {
+//			return;
+//		}
+//		List<UnitDto> sin = findUnitByType(units, QuesStructType.SINGLE_ANSWER_QUESTION);
+//		if (CollectionUtils.isNotEmpty(sin)) {
+//			check(sin, c.getName(), c.getCode());
+//		}
+//		List<UnitDto> mut = findUnitByType(units, QuesStructType.MULTIPLE_ANSWER_QUESTION);
+//		if (CollectionUtils.isNotEmpty(mut)) {
+//			check(mut, c.getName(), c.getCode());
+//		}
+//		List<UnitDto> bool = findUnitByType(units, QuesStructType.BOOL_ANSWER_QUESTION);
+//		if (CollectionUtils.isNotEmpty(bool)) {
+//			check(bool, c.getName(), c.getCode());
+//		}
+
+		List<QuestionDto> quess = findQuestion(rootOrgId, c.getCode());
+		if (CollectionUtils.isEmpty(quess)) {
+			return;
+		}
+		List<QuestionDto> sin = findQuestionByType(quess, QuesStructType.SINGLE_ANSWER_QUESTION);
+		if (CollectionUtils.isNotEmpty(sin)) {
+			checkQuestion(sin, c.getName(), c.getCode());
+		}
+		List<QuestionDto> mut = findQuestionByType(quess, QuesStructType.MULTIPLE_ANSWER_QUESTION);
+		if (CollectionUtils.isNotEmpty(mut)) {
+			checkQuestion(mut, c.getName(), c.getCode());
+		}
+		List<QuestionDto> bool = findQuestionByType(quess, QuesStructType.BOOL_ANSWER_QUESTION);
+		if (CollectionUtils.isNotEmpty(bool)) {
+			checkQuestion(bool, c.getName(), c.getCode());
+		}
+		System.out.println("处理完科目：" + c.getCode());
+	}
+
+//	private List<UnitDto> findUnitByType(List<UnitDto> dtos, QuesStructType type) {
+//		List<UnitDto> ret = new ArrayList<>();
+//		for (UnitDto dto : dtos) {
+//			if (type.equals(dto.getQuestionType())) {
+//				ret.add(dto);
+//			}
+//		}
+//		return ret;
+//	}
+
+	private List<QuestionDto> findQuestionByType(List<QuestionDto> dtos, QuesStructType type) {
+		List<QuestionDto> ret = new ArrayList<>();
+		for (QuestionDto dto : dtos) {
+			if (type.equals(dto.getQuestionType())) {
+				ret.add(dto);
+			}
+		}
+		return ret;
+	}
+
+//	private List<PaperDto> findPaper(String rootOrgId, String courseCode, String paperName) {
+//		Query query = new Query();
+//		query.addCriteria(Criteria.where("orgId").is(rootOrgId));
+//		query.addCriteria(Criteria.where("paperType").is(PaperType.IMPORT.name()));
+//		query.addCriteria(Criteria.where("course.code").is(courseCode));
+//		query.addCriteria(Criteria.where("name").regex(".*?\\.*" + paperName + ".*"));
+//		List<PaperDto> ret = this.mongoTemplate.find(query, PaperDto.class, "paper");
+//		return ret;
+//	}
+
+	private List<QuestionDto> findQuestion(String rootOrgId, String courseCode) {
+		Query query = new Query();
+		query.addCriteria(Criteria.where("orgId").is(rootOrgId));
+		query.addCriteria(Criteria.where("course.code").is(courseCode));
+		query.addCriteria(Criteria.where("creationBy").is(646523L));
+		List<QuestionDto> ret = this.mongoTemplate.find(query, QuestionDto.class, "question");
+		return ret;
+	}
+
+//	private List<UnitDto> findUnit(List<ObjectId> paperIds) {
+//		Query query = new Query();
+//		query.addCriteria(Criteria.where("paper.$id").in(paperIds));
+//		List<UnitDto> ret = this.mongoTemplate.find(query, UnitDto.class, "paperDetailUnit");
+//		return ret;
+//	}
+//
+//	private void check(List<UnitDto> dtos, String cname, String ccode) {
+//		Set<String> checkIds = new HashSet<>();
+//		for (UnitDto dto : dtos) {
+//			dto.setExtractText(PaperUtil.getExtractText(dto.getQuestion()));
+//			if (StringUtils.isEmpty(dto.getExtractText())) {
+//				checkIds.add(dto.getId());
+//			}
+//		}
+//		int group = 0;
+//		for (UnitDto dto : dtos) {
+//			if (checkIds.contains(dto.getId())) {
+//				continue;
+//			}
+//			String quesText1 = dto.getExtractText();
+//			boolean hasGroup = false;
+//			for (UnitDto subdto : dtos) {
+//				if (subdto.getId().equals(dto.getId())) {
+//					continue;
+//				}
+//				if (checkIds.contains(subdto.getId())) {
+//					continue;
+//				}
+//				String quesText2 = subdto.getExtractText();
+//				double similarity = StringSimilarityUtils.getSimilarityWithCosinesBySeg(quesText1, quesText2);
+//				if (similarity > 0.95) {
+//					checkIds.add(dto.getId());
+//					checkIds.add(subdto.getId());
+//					if (!hasGroup) {
+//						hasGroup = true;
+//						group++;
+//						RetDto rd = new RetDto(group, ccode, cname, dto.getPaper().getName(),
+//								dto.getQuestionType().getName(), dto.getPaperDetail().getNumber(), dto.getNumber());
+//						addRet(rd);
+//					}
+//					RetDto rd = new RetDto(group, ccode, cname, subdto.getPaper().getName(),
+//							subdto.getQuestionType().getName(), subdto.getPaperDetail().getNumber(),
+//							subdto.getNumber());
+//					addRet(rd);
+//				}
+//			}
+//		}
+//	}
+
+	private void checkQuestion(List<QuestionDto> dtos, String cname, String ccode) {
+		Set<String> checkIds = new HashSet<>();
+		for (QuestionDto dto : dtos) {
+			dto.setExtractText(PaperUtil.getExtractText(dto));
+			if (StringUtils.isEmpty(dto.getExtractText())) {
+				checkIds.add(dto.getId());
+			}
+		}
+        int length = dtos.size();
+		int group = 0;
+		for (int i = 0; i < length - 1; i++) {
+			QuestionDto dto=dtos.get(i);
+			if (checkIds.contains(dto.getId())) {
+				continue;
+			}
+			String quesText1 = dto.getExtractText();
+			boolean hasGroup = false;
+			for (int j = i + 1; j < length; j++) {
+				QuestionDto subdto=dtos.get(j);
+				if (checkIds.contains(subdto.getId())) {
+					continue;
+				}
+				String quesText2 = subdto.getExtractText();
+				double similarity = StringSimilarityUtils.getSimilarityWithCosinesBySeg(quesText1, quesText2);
+				if (similarity > 0.94) {
+					checkIds.add(dto.getId());
+					checkIds.add(subdto.getId());
+					if (!hasGroup) {
+						hasGroup = true;
+						group++;
+						RetDto rd = new RetDto(group, ccode, cname, dto.getQuestionType().getName(),dto.getId());
+						rd.setValid("是");
+						fillInfo(rd);
+						addRet(rd);
+					}
+					RetDto rd = new RetDto(group, ccode, cname, subdto.getQuestionType().getName(),subdto.getId());
+					rd.setValid("否");
+					fillInfo(rd);
+					addRet(rd);
+				}
+			}
+		}
+	}
+
+	private void fillInfo(RetDto dto) {
+		UnitDto u=getUnit(dto.getQuestionId());
+		dto.setPaperName(u.getPaper().getName());
+		dto.setDetailNum(u.getPaperDetail().getNumber());
+		dto.setUnitNum(u.getNumber());
+		dto.setUnitId(u.getId());
+	}
+	
+	private UnitDto getUnit(String questionId) {
+		Query query = new Query();
+		query.addCriteria(Criteria.where("question.$id").in(new ObjectId(questionId)));
+		query.addCriteria(Criteria.where("paperType").is(PaperType.IMPORT.name()));
+		List<UnitDto> ret = this.mongoTemplate.find(query, UnitDto.class, "paperDetailUnit");
+		return ret.get(0);
+	}
+
+}

src/main/java/cn/com/qmth/dp/examcloud/oe/modules/findreduplicatequestion/ExportQuesReduplicateProducer.java

@@ -0,0 +1,119 @@
+package cn.com.qmth.dp.examcloud.oe.modules.findreduplicatequestion;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.poi.xssf.usermodel.XSSFRow;
+import org.apache.poi.xssf.usermodel.XSSFSheet;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.stereotype.Service;
+
+import cn.com.qmth.dp.examcloud.oe.entity.question.Course;
+import cn.com.qmth.dp.examcloud.oe.entity.question.Question;
+import cn.com.qmth.dp.examcloud.oe.excel.ExportUtils;
+import cn.com.qmth.dp.examcloud.oe.multithread.Consumer;
+import cn.com.qmth.dp.examcloud.oe.multithread.Producer;
+import cn.com.qmth.dp.examcloud.oe.util.ResouceUtil;
+
+@Service
+public class ExportQuesReduplicateProducer extends Producer {
+
+	
+	@Autowired
+	private MongoTemplate mongoTemplate;
+	
+	@Override
+	protected void produce(Map<String, Object> param) throws Exception {
+		List<Course> cs = getCourse();
+		if (CollectionUtils.isNotEmpty(cs)) {
+			for (Course c : cs) {
+				offer(c);
+			}
+		}
+		List<RetDto> ret = new ArrayList<>();
+		int index=0;
+		for(Consumer c:getConsumers()) {
+			ret.addAll(c.getRet());
+			index++;
+		}
+//		Collections.sort(ret, new Comparator<CourseQuestionsCountRetDto>() {
+//			@Override
+//			public int compare(CourseQuestionsCountRetDto o1, CourseQuestionsCountRetDto o2) {
+//				String c1 = o1.getCourseCode();
+//				String c2 = o2.getCourseCode();
+//				return c1.compareTo(c2);
+//			}
+//
+//		});
+		FileOutputStream fos = null;
+		try {
+			File file = new File("d:/ret.xlsx");
+			if (file.exists()) {
+				file.delete();
+			}
+			file.createNewFile();
+			fos = new FileOutputStream(file);
+			ExportUtils.makeExcel(RetDto.class, ret, fos);
+		} catch (Exception e) {
+			e.printStackTrace();
+		} finally {
+			if (fos != null) {
+				try {
+					fos.close();
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
+			}
+		}
+		System.out.println("courseCount:" + cs.size());
+		System.out.println("finish! TotalQuestionCount:" + getTotalQuestionCount((String)param.get("rootOrgId")));
+	}
+	
+	
+	private List<Course> getCourse() {
+		List<Course> list=new ArrayList<>();
+		XSSFWorkbook wb = null;
+		try {
+			wb = new XSSFWorkbook(ResouceUtil.getStream("subject_codes.xlsx"));
+			XSSFSheet sheet = wb.getSheetAt(0);
+			int rows = sheet.getLastRowNum();
+			for (int i = 2; i <= rows; i++) {
+				Course c = new Course();
+				XSSFRow row = sheet.getRow(i);
+				String ecCode = row.getCell(0).getStringCellValue().trim();
+				String name = row.getCell(1).getStringCellValue().trim();
+				c.setCode(ecCode);
+				c.setName(name);
+				list.add(c);
+				
+			}
+		} catch (IOException e) {
+			throw new RuntimeException(e);
+		} finally {
+			if (wb != null) {
+				try {
+					wb.close();
+				} catch (IOException e) {
+				}
+			}
+		}
+		return list;
+	}
+	
+	private long getTotalQuestionCount(String rootOrgId) {
+		Query query = new Query();
+		query.addCriteria(Criteria.where("orgId").is(rootOrgId));
+		long count = mongoTemplate.count(query, Question.class, "question");
+		return count;
+	}
+
+}

src/main/java/cn/com/qmth/dp/examcloud/oe/modules/findreduplicatequestion/ExportReduplicateQuestionService.java

@@ -0,0 +1,33 @@
+package cn.com.qmth.dp.examcloud.oe.modules.findreduplicatequestion;
+
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.springframework.stereotype.Service;
+
+import cn.com.qmth.examcloud.web.support.SpringContextHolder;
+
+@Service
+public class ExportReduplicateQuestionService {
+	private String rootOrgId = "17351";
+	private String paperName = "230517";
+	private int threadCount=4;
+
+	public void start() {
+		Date s=new Date();
+		ExportQuesReduplicateProducer pr = SpringContextHolder.getBean(ExportQuesReduplicateProducer.class);
+		try {
+			Map<String, Object> param=new HashMap<>();
+			param.put("rootOrgId", rootOrgId);
+			param.put("paperName", paperName);
+			pr.startDispose(ExportQuesReduplicateConsumer.class, threadCount, param);
+		} catch (Exception e) {
+			throw new RuntimeException(e);
+
+		}
+		Date e=new Date();
+		System.out.println("time:" + (e.getTime()-s.getTime()));
+	}
+
+}

src/main/java/cn/com/qmth/dp/examcloud/oe/modules/findreduplicatequestion/PaperDto.java

@@ -0,0 +1,22 @@
+package cn.com.qmth.dp.examcloud.oe.modules.findreduplicatequestion;
+
+import javax.persistence.Id;
+
+public class PaperDto {
+	@Id
+    private String id;
+	private String name;
+	public String getId() {
+		return id;
+	}
+	public void setId(String id) {
+		this.id = id;
+	}
+	public String getName() {
+		return name;
+	}
+	public void setName(String name) {
+		this.name = name;
+	}
+	
+}

src/main/java/cn/com/qmth/dp/examcloud/oe/modules/findreduplicatequestion/QuesOptionDto.java

@@ -0,0 +1,27 @@
+package cn.com.qmth.dp.examcloud.oe.modules.findreduplicatequestion;
+
+public class QuesOptionDto {
+
+	private String number;
+	private String optionBody;// 选项内容
+
+	public String getNumber() {
+		return number;
+	}
+
+	public void setNumber(String number) {
+		this.number = number;
+	}
+
+	public String getOptionBody() {
+		return optionBody;
+	}
+
+	public void setOptionBody(String optionBody) {
+		this.optionBody = optionBody;
+	}
+
+	public QuesOptionDto() {
+	}
+
+}

src/main/java/cn/com/qmth/dp/examcloud/oe/modules/findreduplicatequestion/QuestionDto.java

@@ -0,0 +1,71 @@
+package cn.com.qmth.dp.examcloud.oe.modules.findreduplicatequestion;
+
+import java.util.List;
+
+import javax.persistence.Id;
+
+import cn.com.qmth.dp.examcloud.oe.enums.question.QuesStructType;
+
+public class QuestionDto {
+
+	@Id
+	protected String id;
+
+	private QuesStructType questionType;// 试题结构类型
+
+	private String quesBody;
+
+	private List<QuesOptionDto> quesOptions;// 试题选项
+
+	private List<QuestionDto> subQuestions;// 子题目,用于套题
+	private String extractText;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(String id) {
+		this.id = id;
+	}
+
+	public QuesStructType getQuestionType() {
+		return questionType;
+	}
+
+	public void setQuestionType(QuesStructType questionType) {
+		this.questionType = questionType;
+	}
+
+	public String getQuesBody() {
+		return quesBody;
+	}
+
+	public void setQuesBody(String quesBody) {
+		this.quesBody = quesBody;
+	}
+
+	public List<QuesOptionDto> getQuesOptions() {
+		return quesOptions;
+	}
+
+	public void setQuesOptions(List<QuesOptionDto> quesOptions) {
+		this.quesOptions = quesOptions;
+	}
+
+	public List<QuestionDto> getSubQuestions() {
+		return subQuestions;
+	}
+
+	public void setSubQuestions(List<QuestionDto> subQuestions) {
+		this.subQuestions = subQuestions;
+	}
+
+	public String getExtractText() {
+		return extractText;
+	}
+
+	public void setExtractText(String extractText) {
+		this.extractText = extractText;
+	}
+
+}

src/main/java/cn/com/qmth/dp/examcloud/oe/modules/findreduplicatequestion/RetDto.java

@@ -0,0 +1,131 @@
+package cn.com.qmth.dp.examcloud.oe.modules.findreduplicatequestion;
+
+import cn.com.qmth.dp.examcloud.oe.excel.ExcelProperty;
+
+public class RetDto {
+	@ExcelProperty(name = "重复分组", width = 15, index = 0)
+	private Integer group;
+	@ExcelProperty(name = "课程代码", width = 15, index = 1)
+	private String courseCode;
+	@ExcelProperty(name = "课程名称", width = 15, index = 2)
+	private String courseName;
+	@ExcelProperty(name = "试卷名称", width = 15, index = 3)
+	private String paperName;
+	@ExcelProperty(name = "题型", width = 15, index = 4)
+	private String type;
+	@ExcelProperty(name = "大题号", width = 15, index = 5)
+	private Integer detailNum;
+	@ExcelProperty(name = "小题号", width = 15, index = 6)
+	private Integer unitNum;
+
+	@ExcelProperty(name = "unit_id", width = 15, index = 7)
+	private String unitId;
+
+	@ExcelProperty(name = "question_id", width = 15, index = 8)
+	private String questionId;
+
+	@ExcelProperty(name = "是否有效", width = 15, index = 9)
+	private String valid;
+
+	public Integer getGroup() {
+		return group;
+	}
+
+	public void setGroup(Integer group) {
+		this.group = group;
+	}
+
+	public String getCourseCode() {
+		return courseCode;
+	}
+
+	public void setCourseCode(String courseCode) {
+		this.courseCode = courseCode;
+	}
+
+	public String getCourseName() {
+		return courseName;
+	}
+
+	public void setCourseName(String courseName) {
+		this.courseName = courseName;
+	}
+
+	public String getPaperName() {
+		return paperName;
+	}
+
+	public void setPaperName(String paperName) {
+		this.paperName = paperName;
+	}
+
+	public String getType() {
+		return type;
+	}
+
+	public void setType(String type) {
+		this.type = type;
+	}
+
+	public Integer getDetailNum() {
+		return detailNum;
+	}
+
+	public void setDetailNum(Integer detailNum) {
+		this.detailNum = detailNum;
+	}
+
+	public Integer getUnitNum() {
+		return unitNum;
+	}
+
+	public void setUnitNum(Integer unitNum) {
+		this.unitNum = unitNum;
+	}
+
+	public RetDto(Integer group, String courseCode, String courseName, String type, String questionId) {
+		super();
+		this.group = group;
+		this.courseCode = courseCode;
+		this.courseName = courseName;
+		this.type = type;
+		this.questionId = questionId;
+	}
+
+	public RetDto(Integer group, String courseCode, String courseName, String paperName, String type, Integer detailNum,
+			Integer unitNum) {
+		super();
+		this.group = group;
+		this.courseCode = courseCode;
+		this.courseName = courseName;
+		this.paperName = paperName;
+		this.type = type;
+		this.detailNum = detailNum;
+		this.unitNum = unitNum;
+	}
+
+	public String getQuestionId() {
+		return questionId;
+	}
+
+	public void setQuestionId(String questionId) {
+		this.questionId = questionId;
+	}
+
+	public String getUnitId() {
+		return unitId;
+	}
+
+	public void setUnitId(String unitId) {
+		this.unitId = unitId;
+	}
+
+	public String getValid() {
+		return valid;
+	}
+
+	public void setValid(String valid) {
+		this.valid = valid;
+	}
+
+}

src/main/java/cn/com/qmth/dp/examcloud/oe/modules/findreduplicatequestion/UnitDto.java

@@ -0,0 +1,82 @@
+package cn.com.qmth.dp.examcloud.oe.modules.findreduplicatequestion;
+
+import javax.persistence.Id;
+
+import cn.com.qmth.dp.examcloud.oe.enums.question.QuesStructType;
+
+public class UnitDto {
+	@Id
+	private String id;
+	private String name;
+	private Integer number;
+	private QuestionDto question;
+	private PaperDto paper;
+	private QuesStructType questionType;
+	private DetailDto paperDetail;
+	private String extractText;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(String id) {
+		this.id = id;
+	}
+
+	public String getName() {
+		return name;
+	}
+
+	public void setName(String name) {
+		this.name = name;
+	}
+
+	public Integer getNumber() {
+		return number;
+	}
+
+	public void setNumber(Integer number) {
+		this.number = number;
+	}
+
+	public QuestionDto getQuestion() {
+		return question;
+	}
+
+	public void setQuestion(QuestionDto question) {
+		this.question = question;
+	}
+
+	public DetailDto getPaperDetail() {
+		return paperDetail;
+	}
+
+	public void setPaperDetail(DetailDto paperDetail) {
+		this.paperDetail = paperDetail;
+	}
+
+	public QuesStructType getQuestionType() {
+		return questionType;
+	}
+
+	public void setQuestionType(QuesStructType questionType) {
+		this.questionType = questionType;
+	}
+
+	public PaperDto getPaper() {
+		return paper;
+	}
+
+	public void setPaper(PaperDto paper) {
+		this.paper = paper;
+	}
+
+	public String getExtractText() {
+		return extractText;
+	}
+
+	public void setExtractText(String extractText) {
+		this.extractText = extractText;
+	}
+
+}

src/main/java/cn/com/qmth/dp/examcloud/oe/util/Calculator.java

@@ -0,0 +1,170 @@
+package cn.com.qmth.dp.examcloud.oe.util;
+
+import java.math.BigDecimal;
+import java.util.List;
+
+import org.apache.commons.collections4.CollectionUtils;
+
+import cn.com.qmth.examcloud.commons.exception.StatusException;
+
+/**
+ * 计算器
+ *
+ * @author
+ * @date 2019年7月30日
+ * @Copyright (c) 2018-? http://qmth.com.cn All Rights Reserved.
+ */
+public class Calculator {
+
+    /**
+     * 加法 保留两位小数
+     * 
+     * @param v1
+     * @param v2
+     * @return
+     */
+    public static double add(double v1, double v2) {
+        return add(v1, v2, 2);
+
+    }
+
+    /**
+     * 加法 保留指定位小数
+     * 
+     * @param v1
+     * @param v2
+     * @param len
+     * @return
+     */
+    public static double add(double v1, double v2, int len) {
+        BigDecimal b1 = new BigDecimal(v1);
+        BigDecimal b2 = new BigDecimal(v2);
+        return b1.add(b2).setScale(len, BigDecimal.ROUND_HALF_UP).doubleValue();
+
+    }
+    
+    /** 加法 保留指定位小数
+     * @param ds
+     * @param len
+     * @return
+     */
+    public static double add(List<Double> ds, int len) {
+        if(CollectionUtils.isEmpty(ds)) {
+            throw new StatusException("数组为空");
+        }
+        BigDecimal ret = new BigDecimal(0.0);
+        for(Double d:ds) {
+            if(d==null) {
+                throw new StatusException("数组元素为空");
+            }
+            ret=ret.add(new BigDecimal(d));
+        }
+        return ret.setScale(len, BigDecimal.ROUND_HALF_UP).doubleValue();
+
+    }
+
+    /**
+     * 减法 保留两位小数
+     * 
+     * @param v1
+     * @param v2
+     * @return
+     */
+    public static double subtract(double v1, double v2) {
+        return subtract(v1, v2, 2);
+
+    }
+
+    /**
+     * 减法 保留指定位小数
+     * 
+     * @param v1
+     * @param v2
+     * @param len
+     * @return
+     */
+    public static double subtract(double v1, double v2, int len) {
+        BigDecimal b1 = new BigDecimal(v1);
+        BigDecimal b2 = new BigDecimal(v2);
+        return b1.subtract(b2).setScale(len, BigDecimal.ROUND_HALF_UP).doubleValue();
+
+    }
+    /**差值绝对值
+     * @param v1
+     * @param v2
+     * @param len
+     * @return
+     */
+    public static double absoluteDiff(double v1, double v2, int len) {
+        BigDecimal b1 = new BigDecimal(v1);
+        BigDecimal b2 = new BigDecimal(v2);
+        if(v1>v2) {
+            return b1.subtract(b2).setScale(len, BigDecimal.ROUND_HALF_UP).doubleValue();
+        }else {
+            return b2.subtract(b1).setScale(len, BigDecimal.ROUND_HALF_UP).doubleValue();
+        }
+
+    }
+    /**
+     * 乘法 保留两位小数
+     * 
+     * @param v1
+     * @param v2
+     * @return
+     */
+    public static double multiply(double v1, double v2) {
+        return multiply(v1, v2, 2);
+
+    }
+
+    /**
+     * 乘法 保留指定位小数
+     * 
+     * @param v1
+     * @param v2
+     * @param len
+     * @return
+     */
+    public static double multiply(double v1, double v2, int len) {
+        BigDecimal b1 = new BigDecimal(v1);
+        BigDecimal b2 = new BigDecimal(v2);
+        return b1.multiply(b2).setScale(len, BigDecimal.ROUND_HALF_UP).doubleValue();
+
+    }
+    
+    
+    /**
+     * 除法 保留两位小数
+     * 
+     * @param v1
+     * @param v2
+     * @param len
+     * @return
+     */
+    public static double divide(double v1, double v2) {
+        return divide(v1, v2, 2);
+    }
+
+    /**
+     * 除法 保留指定位小数
+     * 
+     * @param v1
+     * @param v2
+     * @param len
+     * @return
+     */
+    public static double divide(double v1, double v2, int len) {
+        BigDecimal b1 = new BigDecimal(v1);
+        BigDecimal b2 = new BigDecimal(v2);
+        return b1.divide(b2, len, BigDecimal.ROUND_HALF_UP).doubleValue();
+    }
+
+    public static String divide2String(Double v1, Double v2, int len) {
+        if(v1==null||v2==null||v2==0) {
+            return "-";
+        }
+        BigDecimal b1 = new BigDecimal(v1);
+        BigDecimal b2 = new BigDecimal(v2);
+        return String.valueOf(b1.divide(b2, len, BigDecimal.ROUND_HALF_UP).doubleValue());
+    }
+}

src/main/java/cn/com/qmth/dp/examcloud/oe/util/PaperUtil.java

@@ -0,0 +1,55 @@
+package cn.com.qmth.dp.examcloud.oe.util;
+
+import java.util.List;
+
+import org.apache.commons.lang3.StringUtils;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+
+import cn.com.qmth.dp.examcloud.oe.modules.findreduplicatequestion.QuesOptionDto;
+import cn.com.qmth.dp.examcloud.oe.modules.findreduplicatequestion.QuestionDto;
+
+public class PaperUtil {
+    public static String getExtractText(QuestionDto question) {
+        StringBuilder quesText = new StringBuilder();
+        if (question == null) {
+            return quesText.toString();
+        }
+
+        if (!StringUtils.isEmpty(question.getQuesBody())) {
+            quesText.append(getTextInHtml(question.getQuesBody()));
+        }
+
+        List<QuesOptionDto> quesOptionList = question.getQuesOptions();
+        if (quesOptionList != null && quesOptionList.size() > 0) {
+            for (QuesOptionDto quesOption : quesOptionList) {
+                if (!StringUtils.isEmpty(quesOption.getOptionBody())) {
+                    quesText.append(getTextInHtml(quesOption.getOptionBody()));
+                }
+            }
+        }
+
+        return quesText.toString();
+    }
+    public static String getTextInHtml(String htmlStr) {
+        htmlStr = htmlStr.replaceAll("\\&[a-zA-Z]{1,10};", "").trim();
+        if (!htmlStr.startsWith("<p>")) {
+            return htmlStr;
+        }
+
+        try {
+            org.jsoup.nodes.Document doc = Jsoup.parse(htmlStr);
+            StringBuilder textStr = new StringBuilder();
+            Elements links = doc.select("p").removeAttr("img");
+
+            for (Element link : links) {
+                textStr.append(link.text().trim());
+            }
+
+            return textStr.toString();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

src/main/java/cn/com/qmth/dp/examcloud/oe/util/StringSimilarityUtils.java

@@ -0,0 +1,276 @@
+package cn.com.qmth.dp.examcloud.oe.util;
+
+import org.ansj.domain.Result;
+import org.ansj.domain.Term;
+import org.ansj.splitWord.analysis.ToAnalysis;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.*;
+
+/**
+ * 计算相似度工具包:
+ *
+ * @author songyue
+ * @date 2016-05-11
+ */
+public class StringSimilarityUtils {
+
+    /**
+     * 对输入字符串分词
+     *
+     * @param str
+     * @return ArrayList
+     * @author songyue
+     */
+    public static List<String> segmentText(String str) {
+        List<String> segResult = new ArrayList<>();// 分词结果
+        Result result = ToAnalysis.parse(str);
+        List<Term> terms = result.getTerms();
+        for (Term term : terms) {
+            if (StringUtils.isNotEmpty(term.getName().trim())) {
+                segResult.add(term.getName());
+            }
+        }
+        return segResult;
+    }
+
+    /**
+     * 计算相似度（两个分词集合，分词匹配，算法为余弦定理）
+     *
+     * @param seg1
+     * @param seg2
+     * @return
+     */
+    public static double getSimilarityWithCosinesBySeg(String seg1, String seg2) {
+        double similarity = 0;
+        int size1 = 0;
+        int size2 = 0;
+        seg1 = stringFilter(seg1);
+        seg2 = stringFilter(seg2);
+        List<String> w1 = segmentText(seg1);
+        List<String> w2 = segmentText(seg2);
+        if (w1 != null && (size1 = w1.size()) != 0 && w2 != null && (size2 = w2.size()) != 0) {
+            Map<String, int[]> countMap = new HashMap<>();
+            String index = null;
+            // 将w1与w2分词出现频次统计入coutMap中
+            for (int i = 0; i < size1; i++) {
+                index = w1.get(i);
+                if (index != null) {
+                    int[] c = countMap.get(index);
+                    if (c != null && c.length == 2) {
+                        c[0]++;
+                    } else {
+                        c = new int[2];
+                        c[0] = 1;
+                        c[1] = 0;
+                        countMap.put(index, c);
+                    }
+                }
+            }
+            for (int i = 0; i < size2; i++) {
+                index = w2.get(i);
+                if (index != null) {
+                    int[] c = countMap.get(index);
+                    if (c != null && c.length == 2) {
+                        c[1]++;
+                    } else {
+                        c = new int[2];
+                        c[0] = 0;
+                        c[1] = 1;
+                        countMap.put(index, c);
+                    }
+                }
+            }
+            // 根据余弦定理计算相似度
+            Iterator<String> it = countMap.keySet().iterator();
+            double sum = 0;
+            double s1 = 0;
+            double s2 = 0;
+            while (it.hasNext()) {
+                int[] c = countMap.get(it.next());
+                sum += c[0] * c[1];
+                s1 += c[0] * c[0];
+                s2 += c[1] * c[1];
+            }
+            similarity = sum / Math.sqrt(s1 * s2);
+        } else {
+            return 0;
+        }
+        return similarity;
+    }
+
+    /**
+     * 计算相似度（两个字符串，全字匹配，算法为余弦定理）
+     *
+     * @param w1
+     * @param w2
+     * @return
+     */
+    public static double getSimilarityWithCosinesByWords(String w1, String w2) {
+        double similarity = 0;
+        int size1 = 0;
+        int size2 = 0;
+        w1 = stringFilter(w1);
+        w2 = stringFilter(w2);
+        if (w1 != null && (size1 = w1.length()) != 0 && w2 != null && (size2 = w2.length()) != 0) {
+            Map<Character, int[]> countMap = new HashMap<>();
+            char index;
+            // 将w1与w2所有字符出现频次统计入countMap中
+            for (int i = 0; i < size1; i++) {
+                index = w1.charAt(i);
+                int[] c = countMap.get(index);
+                if (c != null && c.length == 2) {
+                    c[0]++;
+                } else {
+                    c = new int[2];
+                    c[0] = 1;
+                    c[1] = 0;
+                    countMap.put(index, c);
+                }
+            }
+            for (int i = 0; i < size2; i++) {
+                index = w2.charAt(i);
+                int[] c = countMap.get(index);
+                if (c != null && c.length == 2) {
+                    c[1]++;
+                } else {
+                    c = new int[2];
+                    c[0] = 0;
+                    c[1] = 1;
+                    countMap.put(index, c);
+                }
+            }
+            // 根据余弦定理计算相似度
+            Iterator<Character> it = countMap.keySet().iterator();
+            double sum = 0;
+            double s1 = 0;
+            double s2 = 0;
+            while (it.hasNext()) {
+                int[] c = countMap.get(it.next());
+                sum += c[0] * c[1];
+                s1 += c[0] * c[0];
+                s2 += c[1] * c[1];
+            }
+            similarity = sum / Math.sqrt(s1 * s2);
+        } else {
+            throw new NullPointerException("传入的参数为空");
+        }
+        return similarity;
+    }
+
+    /**
+     * 计算相似度（两个字符串，采用优化Dice算法）
+     *
+     * @param w1
+     * @param w2
+     * @return
+     */
+    public static double getSimilarityWithDiceOptByWords(String w1, String w2) {
+        if (w1 == null || w2 == null || w1.length() == 0 || w2.length() == 0)
+            return 0;
+
+        if (w1.equals(w2))
+            return 1;
+
+        if (w1.length() == 1 || w2.length() == 1) {
+            if (w1.equals(w2)) {
+                return 1;
+            } else {
+                return 0;
+            }
+        }
+
+        w1 = stringFilter(w1);
+        w2 = stringFilter(w2);
+
+        final int n = w1.length() - 1;
+        final int[] sPairs = new int[n];
+        for (int i = 0; i <= n; i++)
+            if (i == 0)
+                sPairs[i] = w1.charAt(i) << 16;
+            else if (i == n)
+                sPairs[i - 1] |= w1.charAt(i);
+            else
+                sPairs[i] = (sPairs[i - 1] |= w1.charAt(i)) << 16;
+
+        final int m = w2.length() - 1;
+        final int[] tPairs = new int[m];
+        for (int i = 0; i <= m; i++)
+            if (i == 0)
+                tPairs[i] = w2.charAt(i) << 16;
+            else if (i == m)
+                tPairs[i - 1] |= w2.charAt(i);
+            else
+                tPairs[i] = (tPairs[i - 1] |= w2.charAt(i)) << 16;
+
+        Arrays.sort(sPairs);
+        Arrays.sort(tPairs);
+
+        int matches = 0, i = 0, j = 0;
+        while (i < n && j < m) {
+            if (sPairs[i] == tPairs[j]) {
+                matches += 2;
+                i++;
+                j++;
+            } else if (sPairs[i] < tPairs[j])
+                i++;
+            else
+                j++;
+        }
+        return (double) matches / (n + m);
+    }
+
+    /**
+     * 计算相似度（两个字符串，采用一般Dice算法）
+     *
+     * @param w1
+     * @param w2
+     * @return
+     */
+    public static double getSimilarityWithDiceByWords(String w1, String w2) {
+        double similarity = 0;
+        if (w1 != null && w1.length() != 0 && w2 != null && w2.length() != 0) {
+            if (w1.length() == 1 || w2.length() == 1) {
+                if (w1.equals(w2)) {
+                    return 1;
+                } else {
+                    return 0;
+                }
+            }
+            w1 = stringFilter(w1);
+            w2 = stringFilter(w2);
+            Set<String> nx = new HashSet<>();
+            Set<String> ny = new HashSet<>();
+
+            for (int i = 0; i < w1.length() - 1; i++) {
+                char x1 = w1.charAt(i);
+                char x2 = w1.charAt(i + 1);
+                String tmp = "" + x1 + x2;
+                nx.add(tmp);
+            }
+            for (int j = 0; j < w2.length() - 1; j++) {
+                char y1 = w2.charAt(j);
+                char y2 = w2.charAt(j + 1);
+                String tmp = "" + y1 + y2;
+                ny.add(tmp);
+            }
+            Set<String> intersection = new HashSet<>(nx);
+            intersection.retainAll(ny);
+            double totcombigrams = intersection.size();
+            similarity = (2 * totcombigrams) / (nx.size() + ny.size());
+        }
+        return similarity;
+    }
+
+    /**
+     * 过滤特殊字符
+     *
+     * @param str
+     * @return
+     */
+    public static String stringFilter(String str) {
+        String regEx = "\\&[a-zA-Z]{1,10};|[_`~!@#$%^&*()+=|{}':;',\\[\\].<>/?~！@#￥%……&*（）——+|{}【】‘；：”“’。，、？]";
+        return str.replaceAll(regEx, "").trim();
+    }
+
+}

src/main/resources/application.properties

@@ -9,32 +9,32 @@ spring.main.allow-bean-definition-overriding=true
 #
 # ********** redis config **********
 #
-spring.redis.host=192.168.10.139
+spring.redis.host=localhost
 spring.redis.port=6379
 spring.redis.password=123456
 spring.redis.database=9
 #
 # ********** mongodb config **********
 #
-mguri.hostAndPortGroup=192.168.10.139:27017
-mguri.username=examcloud
-mguri.password=examcloud_123
+mguri.hostAndPortGroup=dds-wz93958e4b3d8b141100.mongodb.rds.aliyuncs.com:3717
+mguri.username=reader
+mguri.password=vqd2uq-5zuxcm-ge4fn3
 mguri.database=admin
 spring.data.mongodb.uri=mongodb://${mguri.username}:${mguri.password}@${mguri.hostAndPortGroup}/${mguri.database}
 # oe
-spring.data.mongodb.grid-fs-database=examcloud_core_oe
-spring.data.mongodb.database=examcloud_core_oe
+#spring.data.mongodb.grid-fs-database=examcloud_core_oe
+#spring.data.mongodb.database=examcloud_core_oe
 # question
-spring.data.mongodb.database=comm_ques_bank
-spring.data.mongodb.grid-fs-database=comm_ques_bank
+spring.data.mongodb.database=comm-ques-bank
+spring.data.mongodb.grid-fs-database=comm-ques-bank
 #
 # ********** mysql config **********
 #
-dsurl.host=192.168.10.201
-dsurl.port=3309
-dsurl.database=exam_cloud_test
+dsurl.host=localhost
+dsurl.port=3306
+dsurl.database=test
 spring.datasource.username=root
-spring.datasource.password=root
+spring.datasource.password=123456
 spring.datasource.url=jdbc:mysql://${dsurl.host}:${dsurl.port}/${dsurl.database}?useUnicode=true&characterEncoding=UTF-8&useSSL=false&serverTimezone=GMT%2b8&rewriteBatchedStatements=true
 spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver
 spring.datasource.type=com.alibaba.druid.pool.DruidDataSource

src/main/resources/log4j2.xml

@@ -2,7 +2,7 @@
 <Configuration status="WARN" monitorInterval="30">
 
     <Properties>
-        <Property name="LOG_LEVEL" value="${sys:log.commonLevel}"/>
+        <Property name="LOG_LEVEL" value="debug"/>
         <Property name="LOG_DIR" value="../logs"/>
         <Property name="LOG_PATTERN">
             %d{yyyy-MM-dd HH:mm:ss.SSS} | %clr{%level} | %X{TRACE_ID} %X{CALLER} | %clr{%c{1.1}:%L}{cyan} | %m%n