数据清理

examcloud-core-oe-student-api-provider/src/main/java/cn/com/qmth/examcloud/core/oe/student/api/controller/ExamDataCleanController.java

@@ -0,0 +1,28 @@
+package cn.com.qmth.examcloud.core.oe.student.api.controller;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.PutMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import cn.com.qmth.examcloud.core.oe.student.service.ExamDataCleanService;
+import cn.com.qmth.examcloud.web.support.ControllerSupport;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
+
+@Api(tags = "考试数据清理")
+@RestController
+@RequestMapping("${app.api.oe.student}/examDataClean")
+public class ExamDataCleanController extends ControllerSupport {
+	
+    @Autowired
+    private ExamDataCleanService examDataCleanService;
+
+    @ApiOperation(value = "清理")
+    @PutMapping("/clean")
+    public void clean(@RequestParam(required = false) @ApiParam(value = "在此日期之前（yyyy-MM-dd HH:mm:ss）") String dateBefore) {
+    	examDataCleanService.cleanData(dateBefore);
+    }
+}

examcloud-core-oe-student-dao/src/main/java/cn/com/qmth/examcloud/core/oe/student/dao/entity/ExamRecordDataEntity.java

@@ -35,6 +35,7 @@ import org.hibernate.annotations.DynamicUpdate;
         @Index(name = "IDX_E_O_E_R_D_003", columnList = "examId"),
         @Index(name = "IDX_E_O_E_R_D_004", columnList = "courseId"),
         @Index(name = "IDX_E_O_E_R_D_005", columnList = "batchNum"),
+        @Index(name = "IDX_E_O_E_R_D_006", columnList = "creationTime"),
 })
 @DynamicInsert
 @DynamicUpdate

examcloud-core-oe-student-service/src/main/java/cn/com/qmth/examcloud/core/oe/student/service/ExamDataCleanService.java

@@ -0,0 +1,15 @@
+package cn.com.qmth.examcloud.core.oe.student.service;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author chenken
+ */
+public interface ExamDataCleanService {
+
+    public void cleanData(String dateBefore);
+
+	void disposeBatch(String dateBefore, Map<String, Integer> conutMap, List<Long> ids);
+
+}

examcloud-core-oe-student-service/src/main/java/cn/com/qmth/examcloud/core/oe/student/service/impl/ExamDataCleanServiceImpl.java

@@ -0,0 +1,149 @@
+package cn.com.qmth.examcloud.core.oe.student.service.impl;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import cn.com.qmth.examcloud.commons.exception.StatusException;
+import cn.com.qmth.examcloud.commons.util.DateUtil;
+import cn.com.qmth.examcloud.commons.util.ThreadLocalUtil;
+import cn.com.qmth.examcloud.core.oe.student.service.ExamDataCleanService;
+import cn.com.qmth.examcloud.support.Constants;
+import cn.com.qmth.examcloud.support.redis.RedisKeyHelper;
+import cn.com.qmth.examcloud.web.redis.RedisClient;
+import cn.com.qmth.examcloud.web.support.SpringContextHolder;
+
+@Service("examDataCleanService")
+public class ExamDataCleanServiceImpl implements ExamDataCleanService {
+	private static final Logger log = LoggerFactory.getLogger(ExamDataCleanService.class);
+
+	private static final String logPrefix = "ExamDataClean ";
+
+	@Autowired
+	private RedisClient redisClient;
+
+	@Autowired
+	private JdbcTemplate jdbcTemplate;
+
+	@Override
+	public void cleanData(String dateBefore) {
+		if (StringUtils.isNotBlank(dateBefore)) {
+			try {
+				DateUtil.parse(dateBefore, DateUtil.DatePatterns.CHINA_DEFAULT);
+			} catch (Exception e) {
+				throw new StatusException("日期格式错误,正确格式:" + DateUtil.DatePatterns.CHINA_DEFAULT);
+			}
+		}
+		if (redisClient.setIfAbsent(Constants.EXAM_DATA_CLEAN_LOCK_PREFIX, ThreadLocalUtil.getTraceId(), 60)) {
+			try {
+				doLog("start");
+				diposeClean(dateBefore);
+			} finally {
+				redisClient.delete(Constants.EXAM_DATA_CLEAN_LOCK_PREFIX);
+				doLog("finish");
+			}
+		} else {
+			throw new StatusException("有正在进行的数据清理操作,稍后再试");
+		}
+	}
+
+	private void diposeClean(String dateBefore) {
+		Long startId = -1L;
+		Map<String, Integer> conutMap = new LinkedHashMap<>();
+		conutMap.put("ec_oes_exam_continued_record", 0);
+		conutMap.put("ec_oes_exam_face_biopsy", 0);
+		conutMap.put("ec_oes_exam_face_biopsy_item", 0);
+		conutMap.put("ec_oes_exam_face_biopsy_item_step", 0);
+		conutMap.put("ec_oes_exam_face_liveness_verify", 0);
+		conutMap.put("ec_oes_exam_process_record", 0);
+		conutMap.put("ec_oes_exam_record_data", 0);
+		ExamDataCleanService serv = SpringContextHolder.getBean(ExamDataCleanService.class);
+		for (;;) {
+			List<Long> ids = findRecIds(startId, dateBefore);
+			if (ids == null || ids.size() == 0) {
+				break;
+			}
+			startId = ids.get(ids.size() - 1);
+			serv.disposeBatch(dateBefore, conutMap, ids);
+		}
+		for (String k : conutMap.keySet()) {
+			doLog(k + " delete count:" + conutMap.get(k));
+		}
+	}
+
+	@Transactional
+	@Override
+	public void disposeBatch(String dateBefore, Map<String, Integer> conutMap, List<Long> ids) {
+		for (Long examRecordDataId : ids) {
+			cleanCache(examRecordDataId);
+		}
+		diposeCleanSubTable("ec_oes_exam_continued_record", conutMap, ids);
+		diposeCleanSubTable("ec_oes_exam_face_biopsy", conutMap, ids);
+		diposeCleanSubTable("ec_oes_exam_face_biopsy_item", conutMap, ids);
+		diposeCleanSubTable("ec_oes_exam_face_biopsy_item_step", conutMap, ids);
+		diposeCleanSubTable("ec_oes_exam_face_liveness_verify", conutMap, ids);
+		diposeCleanSubTable("ec_oes_exam_process_record", conutMap, ids);
+		diposeRecordTable(conutMap, ids);
+	}
+
+	private void cleanCache(Long examRecordDataId) {
+		String examingActiveTimeKey = RedisKeyHelper.getBuilder().examingActiveTimeKey(examRecordDataId);
+		String examingHeartbeatKey = RedisKeyHelper.getBuilder().examingHeartbeatKey(examRecordDataId);
+		String examRecordDataKey = RedisKeyHelper.getBuilder().examRecordDataKey(examRecordDataId);
+		String studentPaperKey = RedisKeyHelper.getBuilder().studentPaperKey(examRecordDataId);
+		redisClient.delete(examingActiveTimeKey);
+		redisClient.delete(examingHeartbeatKey);
+		redisClient.delete(examRecordDataKey);
+		redisClient.delete(studentPaperKey);
+	}
+
+	private void diposeCleanSubTable(String tb, Map<String, Integer> conutMap, List<Long> ids) {
+		StringBuilder querySql = new StringBuilder().append(" delete from ").append(tb)
+				.append(" where exam_record_data_id in (");
+		querySql.append(StringUtils.join(ids, ","));
+		querySql.append(")");
+		int count = jdbcTemplate.update(querySql.toString());
+		conutMap.put(tb, conutMap.get(tb) + count);
+		refreshLock();
+	}
+
+	private void diposeRecordTable(Map<String, Integer> conutMap, List<Long> ids) {
+		String tb = "ec_oes_exam_record_data";
+		StringBuilder querySql = new StringBuilder().append(" delete from ec_oes_exam_record_data where id in(");
+		querySql.append(StringUtils.join(ids, ","));
+		querySql.append(")");
+		int count = jdbcTemplate.update(querySql.toString());
+		conutMap.put(tb, conutMap.get(tb) + count);
+		refreshLock();
+	}
+
+	private List<Long> findRecIds(Long startId, String dateBefore) {
+		StringBuilder querySql = new StringBuilder()
+				.append(" select id from  ec_oes_exam_record_data  where sync_status='SYNCED' and id>").append(startId);
+		if (StringUtils.isNotBlank(dateBefore)) {
+			querySql.append(" and creation_time <= STR_TO_DATE('").append(dateBefore)
+					.append("', '%Y-%m-%d %H:%i:%s') ");
+		}
+		querySql.append(" order by id limit 100");
+		List<Long> ids = jdbcTemplate.queryForList(querySql.toString(), Long.class);
+		refreshLock();
+		return ids;
+	}
+
+	private void doLog(String msg) {
+		log.warn(logPrefix + msg);
+	}
+	
+	private void refreshLock() {
+		redisClient.expire(Constants.EXAM_DATA_CLEAN_LOCK_PREFIX, 60);
+	}
+
+}