package cn.com.qmth.export; import java.io.File; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.nodes.Node; import com.alibaba.fastjson.JSONObject; public class MyConsumer extends Consumer { private String paperSuff = "(211)"; private int maxqc = 100; private String excelDir = "D:\\kd_export\\"; private String picFiles = "D:\\files\\"; private Pattern mp3Pat = Pattern.compile("[^<]*"); private Pattern imgPat = Pattern.compile(""); private Pattern notmp3Pat = Pattern.compile("([^<]*)"); private List qdb = new ArrayList<>(); private List qzip = new ArrayList<>(); private List adb = new ArrayList<>(); private List azip = new ArrayList<>(); private List http = new ArrayList<>(); private List file = new ArrayList<>(); private Set noques = new HashSet<>(); private Set invalidAnswerCode = new HashSet<>(); @Override public void consume(String code) { Connection connect = null; try { Class.forName("com.mysql.cj.jdbc.Driver"); String url = "jdbc:mysql://localhost:3306/moodle_question?serverTimezone=GMT%2B8"; String user = "root"; String password = "123456"; connect = DriverManager.getConnection(url, user, password); exportPaper(connect, code); ExportByCourseCode.addDisposeCount(); } catch (Exception e) { throw new RuntimeException(e); } finally { if (connect != null) { try { connect.close(); } catch (SQLException e) { } } } } private Course getCourse(Connection connect, String code) throws SQLException, IOException { Course c = new Course(); PreparedStatement preState = null; ResultSet resultSet = null; try { String sql = "select * from mdl_course where idnumber like '" + code + "_%';"; preState = connect.prepareStatement(sql); resultSet = preState.executeQuery(); int count = 0; while (resultSet.next()) { count++; if (count > 1) { return null; } c.setId(resultSet.getLong("id")); c.setName(resultSet.getString("shortname")); c.setIdnumber(resultSet.getString("idnumber")); c.setCode(code); } return c; } finally { if (resultSet != null) { resultSet.close(); } if (preState != null) { preState.close(); } } } private List getContextIds(Connection connect, Long courseId) throws SQLException, IOException { List ids = new ArrayList<>(); PreparedStatement preState = null; ResultSet resultSet = null; try { String sql = "select * from mdl_context where instanceid =" + courseId + " and contextlevel=50 order by id "; preState = connect.prepareStatement(sql); resultSet = preState.executeQuery(); while (resultSet.next()) { ids.add(resultSet.getLong("id")); } return ids; } finally { if (resultSet != null) { resultSet.close(); } if (preState != null) { preState.close(); } } } private List getQuestionCategorie(Connection connect, List cids) throws SQLException, IOException { List ids = new ArrayList<>(); PreparedStatement preState = null; ResultSet resultSet = null; try { String sql = "select * from mdl_question_categories where contextid in (" + getInStr(cids) + ") order by id "; preState = connect.prepareStatement(sql); resultSet = preState.executeQuery(); while (resultSet.next()) { ids.add(resultSet.getLong("id")); } return ids; } finally { if (resultSet != null) { resultSet.close(); } if (preState != null) { preState.close(); } } } private List getQuestion(Connection connect, List qcids) throws SQLException, IOException { List qs = new ArrayList<>(); PreparedStatement preState = null; ResultSet resultSet = null; try { String sql = "SELECT f.* FROM mdl_question f WHERE category IN (" + getInStr(qcids) + ") AND f.qtype IN ('multichoice','truefalse') "; preState = connect.prepareStatement(sql); resultSet = preState.executeQuery(); while (resultSet.next()) { KdQuestion q = new KdQuestion(); q.setId(resultSet.getLong("id")); q.setBody(disBody(resultSet.getString("questiontext"))); q.setQtype(resultSet.getString("qtype")); q.setObjective(true); qs.add(q); } return qs; } finally { if (resultSet != null) { resultSet.close(); } if (preState != null) { preState.close(); } } } private List getAnswer(Connection connect, KdQuestion q) throws SQLException, IOException { List as = new ArrayList<>(); PreparedStatement preState = null; ResultSet resultSet = null; try { String sql = "select * from mdl_question_answers where question =" + q.getId(); preState = connect.prepareStatement(sql); resultSet = preState.executeQuery(); while (resultSet.next()) { Answer a = new Answer(); a.setId(resultSet.getLong("id")); a.setBody(resultSet.getString("answer")); a.setScore(resultSet.getDouble("fraction")); as.add(a); } return as; } finally { if (resultSet != null) { resultSet.close(); } if (preState != null) { preState.close(); } } } private void exportPaper(Connection connect, String code) throws Exception { Course c = getCourse(connect, code); if (c == null) { noques.add(code); return; } List ctids = getContextIds(connect, c.getId()); if (ctids.size() == 0) { noques.add(code); return; } List qcids = getQuestionCategorie(connect, ctids); if (qcids.size() == 0) { noques.add(code); return; } List qs = getQuestion(connect, qcids); if (qs.size() == 0) { noques.add(code); return; } for (KdQuestion q : qs) { List as = getAnswer(connect, q); disposeAnswer(q, as); } qs = removeInvalidQuestion(qs, code); if (qs.size() == 0) { noques.add(code); return; } File courseCode = new File(excelDir + c.getCode() + "\\"); courseCode.mkdirs(); File att = new File(excelDir + c.getCode() + "\\att\\"); att.mkdirs(); for (KdQuestion q : qs) { try { disposeMedia(connect, c.getIdnumber(), q, att); q.setValid(true); } catch (MediaNotFoundException e) { q.setValid(false); } } List tem = new ArrayList<>(); for (KdQuestion q : qs) { if (q.getValid()) { tem.add(q); } } qs = tem; if (qs.size() == 0) { noques.add(code); return; } List single = new ArrayList<>(); List muti = new ArrayList<>(); List boo = new ArrayList<>(); for (KdQuestion q : qs) { if (q.getStructType() == 1) { single.add(q); } else if (q.getStructType() == 2) { muti.add(q); } else if (q.getStructType() == 3) { boo.add(q); } } createPapers(single, c, 1); createPapers(muti, c, 2); createPapers(boo, c, 3); // int detailIndx = 0; // List des = new ArrayList<>(); // if (single.size() > 0) { // detailIndx++; // KdDetail d = new KdDetail(); // d.setName("单选题"); // d.setNumber(detailIndx); // d.setQuestionCount(single.size()); // d.setTotalScore((double) single.size()); // d.setQuestions(single); // des.add(d); // } // if (muti.size() > 0) { // detailIndx++; // KdDetail d = new KdDetail(); // d.setName("多选题"); // d.setNumber(detailIndx); // d.setQuestionCount(muti.size()); // d.setTotalScore((double) muti.size()); // d.setQuestions(muti); // des.add(d); // } // if (boo.size() > 0) { // detailIndx++; // KdDetail d = new KdDetail(); // d.setName("判断题"); // d.setNumber(detailIndx); // d.setQuestionCount(boo.size()); // d.setTotalScore((double) boo.size()); // d.setQuestions(boo); // des.add(d); // } // paper.setDetails(des); // paper.setDetailCount(detailIndx); // // FileUtil.writeFile(courseCode.getAbsolutePath(), "\\paper.json", JSONObject.toJSONString(paper)); } private void createPapers(List qs, Course c, int structType) throws IOException { if (qs == null || qs.size() == 0) { return; } if (qs.size() <= maxqc) { createPaper(qs, c, structType, 1); } else { int size = qs.size(); int len = maxqc; int count = (size + len - 1) / len; for (int i = 0; i < count; i++) { List subList = qs.subList(i * len, ((i + 1) * len > size ? size : len * (i + 1))); createPaper(subList, c, structType, i + 1); } } } private void createPaper(List qs, Course c, int structType, int indx) throws IOException { if (qs.size() == 0) { return; } String detailName=""; if (structType == 1) { detailName="单选题"; } else if (structType == 2) { detailName="多选题"; } else if (structType == 3) { detailName="判断题"; } KdPaper paper = new KdPaper(); paper.setTotalScore((double) qs.size()); paper.setName(c.getName()+paperSuff+detailName+"_"+indx); paper.setCourseCode(c.getCode()); List des = new ArrayList<>(); KdDetail d = new KdDetail(); d.setName(detailName); d.setNumber(1); d.setQuestionCount(qs.size()); d.setTotalScore((double) qs.size()); d.setQuestions(qs); des.add(d); paper.setDetails(des); paper.setDetailCount(1); File paperdir = new File(excelDir + c.getCode() + "\\paper"+structType+"_"+indx+"\\"); paperdir.mkdirs(); FileUtil.writeFile(paperdir.getAbsolutePath(), "\\paper.json", JSONObject.toJSONString(paper)); } private List removeInvalidQuestion(List qs, String courseCode) { List ret = new ArrayList<>(); for (KdQuestion q : qs) { if (StringUtils.isBlank(q.getAnswer())) { invalidAnswerCode.add(q.getId()); continue; } if (isEmpty(q.getBody())) { invalidAnswerCode.add(q.getId()); continue; } if (q.getStructType() == 1 || q.getStructType() == 2) { if (!checkQuestionAndRemoveInvalidOption(q)) { invalidAnswerCode.add(q.getId()); continue; } } ret.add(q); } return ret; } private boolean checkQuestionAndRemoveInvalidOption(KdQuestion q) { List ret = new ArrayList<>(); for (KdQuesOption op : q.getOptions()) { if (isEmpty(op.getBody())) { if (op.getSelect()) { return false; } } else { ret.add(op); } } if (ret.size() < 2) { return false; } else { q.setOptions(ret); } return true; } private void disposeAnswer(KdQuestion q, List as) { if (CollectionUtils.isEmpty(as)) { return; } if ("truefalse".equals(q.getQtype())) { q.setStructType(3); for (Answer a : as) { if (a.getScore() > 0) { q.setAnswer("对".equals(a.getBody().trim()) ? "true" : "false"); return; } } } else { int index = 0; List ops = new ArrayList<>(); StringBuilder sb = new StringBuilder(); for (Answer a : as) { index++; KdQuesOption op = new KdQuesOption(); op.setAnswerId(a.getId()); op.setNumber(index); op.setBody(disBody(a.getBody())); if (a.getScore() > 0) { op.setSelect(true); sb.append(index).append(","); } else { op.setSelect(false); } ops.add(op); } q.setOptions(ops); if (sb.length() > 0) { sb.deleteCharAt(sb.length() - 1); } if (sb.indexOf(",") > 0) { q.setStructType(2); } else { q.setStructType(1); } q.setAnswer(sb.toString()); } } private String disBody(String html) { Document doc = Jsoup.parse(html); Element b = doc.body(); if (b.childrenSize() == 1 && "p".equals(b.child(0).nodeName())) { b.child(0).tagName("span"); } if (b.childrenSize() > 0) { for (Element n : b.children()) { if ("p".equals(n.nodeName())) { boolean[] find = new boolean[] { false }; findImg(n, find); if (!find[0] && StringUtils.isBlank(n.text())) { n.tagName("span"); } } disNode(n); } } return b.html(); } private void findImg(Element e, boolean[] find) { if (find[0]) { return; } else { if ("img".equals(e.nodeName())) { find[0] = true; return; } else { if (e.childrenSize() > 0) { for (Element ce : e.children()) { findImg(ce, find); } } else { return; } } } } private static void disNode(Node n) { if (n instanceof Element) { if (!"img".equals(n.nodeName())) { n.removeAttr("style"); } if ("h1".equals(n.nodeName()) || "h2".equals(n.nodeName()) || "h3".equals(n.nodeName()) || "h4".equals(n.nodeName()) || "h5".equals(n.nodeName())) { ((Element) n).tagName("h6"); } if ("strong".equals(n.nodeName()) || "a".equals(n.nodeName()) || "u".equals(n.nodeName()) || "em".equals(n.nodeName())) { ((Element) n).tagName("span"); } if (n.childNodeSize() > 0) { for (Node cn : n.childNodes()) { disNode(cn); } } } } private String getInStr(List cids) { StringBuilder sb = new StringBuilder(); for (Long id : cids) { sb.append(id).append(","); } sb.deleteCharAt(sb.length() - 1); return sb.toString(); } // private Map findAtag(String body) { // Map set = new HashMap<>(); // if (StringUtils.isBlank(body)) { // return set; // } // Matcher m = notmp3Pat.matcher(body); // while (m.find()) { // String f = m.group(1); // set.put(m.group(), f); // } // return set; // } // // private void removeATag(KdQuestion q) throws Exception { // String body = q.getBody(); // // Map as = findAtag(body); // if (as.size() > 0) { // for (String k : as.keySet()) { // String val = as.get(k); // body = body.replaceAll(k, val); // } // q.setBody(body); // } // // if (CollectionUtils.isNotEmpty(q.getOptions())) { // for (KdQuesOption o : q.getOptions()) { // String obody = o.getBody(); // // Map oas = findAtag(obody); // if (oas.size() > 0) { // for (String k : oas.keySet()) { // String val = oas.get(k); // obody = obody.replaceAll(k, val); // } // o.setBody(obody); // } // } // } // } private void disposeMedia(Connection connect, String courseidnumber, KdQuestion q, File att) throws Exception { String body = q.getBody(); Map imgs = findAllImg(body, q.getId(), null, courseidnumber); if (imgs.size() > 0) { for (String k : imgs.keySet()) { String img = imgs.get(k); String fileName = getFileName(img); File file = getQuestionFile(connect, q.getId(), courseidnumber, fileName); if (file == null) { ExportByCourseCode.addNotFd(); // body = body.replaceAll(k, "[未找到图片文件]"); throw new MediaNotFoundException(); } else { ExportByCourseCode.addFd(); body = body.replaceAll(img.replaceAll("\\?", "\\\\?"), FileUtil.fileToBase64Src(file)); } } q.setBody(body); } Map audios = findAllAudio(body, q.getId(), null, courseidnumber); if (audios.size() > 0) { q.setHaveAudio(true); for (String k : audios.keySet()) { String val = audios.get(k); String fileName = getFileName(val); File file = getQuestionFile(connect, q.getId(), courseidnumber, fileName); if (file == null) { ExportByCourseCode.addNotFd(); // body = body.replaceAll(k, "[未找到音频文件]"); throw new MediaNotFoundException(); } else { ExportByCourseCode.addFd(); File newAudio = new File(att.getAbsoluteFile() + "\\" + file.getName()); newAudio.createNewFile(); FileUtils.copyFile(file, newAudio); body = body.replaceAll(k.replaceAll("\\?", "\\\\?"), ""); } } q.setBody(body); } if (CollectionUtils.isNotEmpty(q.getOptions())) { for (KdQuesOption o : q.getOptions()) { String obody = o.getBody(); Map oimgs = findAllImg(obody, null, o.getAnswerId(), courseidnumber); if (oimgs.size() > 0) { for (String k : oimgs.keySet()) { String img = oimgs.get(k); String fileName = getFileName(img); File file = getAnswerFile(connect, o.getAnswerId(), courseidnumber, fileName); if (file == null) { ExportByCourseCode.addNotFd(); // obody = obody.replaceAll(k, "[未找到图片文件]"); throw new MediaNotFoundException(); } else { ExportByCourseCode.addFd(); obody = obody.replaceAll(img.replaceAll("\\?", "\\\\?"), FileUtil.fileToBase64Src(file)); } } o.setBody(obody); } Map oaudios = findAllAudio(obody, null, o.getAnswerId(), courseidnumber); if (oaudios.size() > 0) { q.setHaveAudio(true); for (String k : oaudios.keySet()) { String val = oaudios.get(k); String fileName = getFileName(val); File file = getAnswerFile(connect, o.getAnswerId(), courseidnumber, fileName); if (file == null) { ExportByCourseCode.addNotFd(); // obody = obody.replaceAll(k, "[未找到音频文件]"); throw new MediaNotFoundException(); } else { ExportByCourseCode.addFd(); File newAudio = new File(att.getAbsoluteFile() + "\\" + file.getName()); newAudio.createNewFile(); FileUtils.copyFile(file, newAudio); obody = obody.replaceAll(k.replaceAll("\\?", "\\\\?"), ""); } } o.setBody(obody); } } } } private Map findAllImg(String body, Long qid, Long aid, String courseidnumber) { Map set = new HashMap<>(); Matcher m = imgPat.matcher(body); while (m.find()) { String f = m.group(1); if (f.startsWith("file:") || f.startsWith("FILE:")) { if (qid != null) file.add("questionid:" + qid + " " + courseidnumber + " " + f); if (aid != null) file.add("answerId:" + aid + " " + courseidnumber + " " + f); continue; } if (f.startsWith("http:") || f.startsWith("HTTP:")) { if (qid != null) http.add("questionid:" + qid + " " + courseidnumber + " " + f); if (aid != null) http.add("answerId:" + aid + " " + courseidnumber + " " + f); continue; } if (!f.startsWith("data:image")) { set.put(m.group(), f); } } return set; } private Map findAllAudio(String body, Long qid, Long aid, String courseidnumber) { Map set = new HashMap<>(); Matcher m = mp3Pat.matcher(body); while (m.find()) { String f = m.group(1); if (f.startsWith("file:") || f.startsWith("FILE:")) { if (qid != null) file.add("questionid:" + qid + " " + courseidnumber + " " + f); if (aid != null) file.add("answerId:" + aid + " " + courseidnumber + " " + f); continue; } if (f.startsWith("http:") || f.startsWith("HTTP:")) { if (qid != null) http.add("questionid:" + qid + " " + courseidnumber + " " + f); if (aid != null) http.add("answerId:" + aid + " " + courseidnumber + " " + f); continue; } set.put(m.group(), f); } return set; } private boolean hasImg(String body) { Matcher m = imgPat.matcher(body); while (m.find()) { return true; } return false; } private boolean hasAudio(String body) { Matcher m = mp3Pat.matcher(body); while (m.find()) { return true; } return false; } private boolean isEmpty(String html) { if (StringUtils.isBlank(html)) { return true; } if (hasImg(html) || hasAudio(html)) { return false; } Document doc = Jsoup.parse(html); Element b = doc.body(); if (StringUtils.isBlank(b.wholeText())) { return true; } else { return false; } } private String getFileName(String s) throws UnsupportedEncodingException { int end = s.length(); int f = s.indexOf("?"); if (f != -1) { end = f; } return URLDecoder.decode(s.substring(s.lastIndexOf("/") + 1, end), "utf-8"); } private File getQuestionFile(Connection connect, Long quesId, String courseidnumber, String fileName) throws Exception { PreparedStatement preState = null; ResultSet resultSet = null; try { String hash = null; String sql = " select * from mdl_files where itemid = " + quesId + " and source='" + fileName + "' and component = 'question' and filearea = 'questiontext' "; preState = connect.prepareStatement(sql); resultSet = preState.executeQuery(); while (resultSet.next()) { hash = resultSet.getString("contenthash"); break; } if (hash == null) { qdb.add(quesId + " " + courseidnumber + " " + fileName); return null; } String suff = fileName.substring(fileName.lastIndexOf(".")); File file = new File(picFiles + courseidnumber + "\\" + hash + suff); if (!file.exists()) { qzip.add(quesId + " " + courseidnumber + " " + fileName); return null; } return file; } finally { if (resultSet != null) { resultSet.close(); } if (preState != null) { preState.close(); } } } private File getAnswerFile(Connection connect, Long answerId, String courseidnumber, String fileName) throws Exception { PreparedStatement preState = null; ResultSet resultSet = null; try { String hash = null; String sql = " select * from mdl_files where itemid = " + answerId + " and source='" + fileName + "' and component = 'question' and filearea = 'answer' "; preState = connect.prepareStatement(sql); resultSet = preState.executeQuery(); while (resultSet.next()) { hash = resultSet.getString("contenthash"); break; } if (hash == null) { adb.add(answerId + " " + courseidnumber + " " + fileName); return null; } String suff = fileName.substring(fileName.lastIndexOf(".")); File file = new File(picFiles + courseidnumber + "\\" + hash + suff); if (!file.exists()) { azip.add(answerId + " " + courseidnumber + " " + fileName); return null; } return file; } finally { if (resultSet != null) { resultSet.close(); } if (preState != null) { preState.close(); } } } @Override public void initResult() { setResult(new HashMap<>()); getResult().put("qdb", qdb); getResult().put("qzip", qzip); getResult().put("adb", adb); getResult().put("azip", azip); getResult().put("invalidAnswerCode", invalidAnswerCode); getResult().put("file", file); getResult().put("http", http); getResult().put("noques", noques); } }