前一段时间,公司让用Lucene检索文档,自己写了些代码,在这里记录一下,以免忘记了。
其实,简单的Lucene的入门还是很简单的,它的整体构造和关系型数据库差不多,一个键对应一个值,生成索引,然后根据索引去查找文档内容,在将内容通过别的方式显示出来。
Lucene创建、增、删、改索引:
package com.haiyisoft.szgl.file.service.impl; import java.io.File; /** * 档案管理的创建索引 * * @author haojiahong * * <p>Modification History:</p> * <p>Date Author Description</p> * <p>--------------------------------------------------------------</p> * <p>20151027 haojiahong new</p> * <p> </p> */ @Component("schDocForDocBuilderService") public class SchDocForDocBuilderServiceImpl implements SchDocForDocBuilderService { @Autowired public DocService docService; @Autowired public FileContentService fileContentService; private long time = 0; /** * 创建lucene索引 */ public void creatLucene() { IndexWriter indexWriter = null; try { File indexDir = new File(FileManage.searchCenterForDocPath); creatFile(indexDir); delAllFile(indexDir); Directory dir = FSDirectory.open(indexDir); Analyzer luceneAnalyzer = new IKAnalyzer(); IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_36, luceneAnalyzer); indexWriter = new IndexWriter(dir, iwc); LogUtil.getAppLoger().debug("开始创建索引"); long indexcount = this.createIndex(indexWriter); LogUtil.getAppLoger().debug("创建索引结束,共处理数据行数" + indexcount + "条"); indexWriter.commit(); indexWriter.close(); } catch (IOException ex) { ex.printStackTrace(); } } /** * 按照数据集创建索引 * @param indexWriter * @return */ private long createIndex(IndexWriter indexWriter) { try { this.showtime(); long current = 0; // current += this.initFile(indexWriter);//根据文档建立索引 current += this.initFileWithDocument(indexWriter);// 根据档案建立索引 return current; } catch (Exception e) { e.printStackTrace(); return -1; } } private long initFileWithDocument(IndexWriter indexWriter) { long current = 0; String jpql = "select file from FileManage file where 1=1"; List<FileManage> fmLs = (List) JPAUtil.find(jpql); for (FileManage fm : fmLs) { try { indexWriter.addDocument(initLuceneDocument(indexWriter, current, fm)); current++; } catch (Exception e) { e.printStackTrace(); } } return current; } /** * 增量添加 lucene索引 */ @Override public void insertLucene(String fmUuid) { IndexWriter indexWriter = null; try { File indexDir = new File(FileManage.searchCenterForDocPath); Directory dir = FSDirectory.open(indexDir); Analyzer luceneAnalyzer = new IKAnalyzer(); IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_36, luceneAnalyzer); indexWriter = new IndexWriter(dir, iwc); LogUtil.getAppLoger().debug("开始增量添加索引"); long indexcount = this.insertIndex(indexWriter, fmUuid); LogUtil.getAppLoger().debug("添加索引结束,共处理数据行数" + indexcount + "条"); indexWriter.commit(); indexWriter.close(); } catch (IOException ex) { ex.printStackTrace(); } } /** * 增量添加索引 * @param indexWriter * @return */ private long insertIndex(IndexWriter indexWriter, String fmUuid) { try { this.showtime(); long current = 0; current += this.insertFileWithDocument(indexWriter, fmUuid);// 根据档案添加索引 return current; } catch (Exception e) { e.printStackTrace(); return -1; } } private long insertFileWithDocument(IndexWriter indexWriter, String fmUuid) { long current = 0; FileManage fm = JPAUtil.loadById(FileManage.class, fmUuid); try { indexWriter.addDocument(initLuceneDocument(indexWriter, current, fm)); current++; } catch (Exception e) { e.printStackTrace(); } return current; } /** * 更新Lucene索引 */ @Override public void updateLucene(String fmUuid) { IndexWriter indexWriter = null; try { File indexDir = new File(FileManage.searchCenterForDocPath); Directory dir = FSDirectory.open(indexDir); Analyzer luceneAnalyzer = new IKAnalyzer(); IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_36, luceneAnalyzer); indexWriter = new IndexWriter(dir, iwc); LogUtil.getAppLoger().debug("开始更新索引"); long indexcount = this.updateIndex(indexWriter, fmUuid); LogUtil.getAppLoger().debug("更新索引结束,共处理数据行数" + indexcount + "条"); indexWriter.commit(); indexWriter.close(); } catch (IOException ex) { ex.printStackTrace(); } } /** * 更新Lucene索引 */ private long updateIndex(IndexWriter indexWriter, String fmUuid) { try { this.showtime(); long current = 0; current += this.updateFileWithDocument(indexWriter, fmUuid);// 根据档案更新索引 return current; } catch (Exception e) { e.printStackTrace(); return -1; } } private long updateFileWithDocument(IndexWriter indexWriter, String fmUuid) { long current = 0; FileManage fm = JPAUtil.loadById(FileManage.class, fmUuid); try { indexWriter.updateDocument(new Term("UUID", fmUuid), initLuceneDocument(indexWriter, current, fm)); current++; } catch (Exception e) { e.printStackTrace(); } return current; } /** * 删除Lucene索引 */ @Override public void deteleLucene(String fmUuid) { IndexWriter indexWriter = null; try { File indexDir = new File(FileManage.searchCenterForDocPath); Directory dir = FSDirectory.open(indexDir); Analyzer luceneAnalyzer = new IKAnalyzer(); IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_36, luceneAnalyzer); indexWriter = new IndexWriter(dir, iwc); LogUtil.getAppLoger().debug("开始删除索引"); long indexcount = this.deteleIndex(indexWriter, fmUuid); LogUtil.getAppLoger().debug("删除索引结束,共处理数据行数" + indexcount + "条"); indexWriter.commit(); indexWriter.close(); } catch (IOException ex) { ex.printStackTrace(); } } /** * 删除Lucene索引 * @param indexWriter * @param fmUuid * @return */ private long deteleIndex(IndexWriter indexWriter, String fmUuid) { try { this.showtime(); long current = 0; current += this.deleteFileWithDocument(indexWriter, fmUuid);// 根据档案删除索引 return current; } catch (Exception e) { e.printStackTrace(); return -1; } } private long deleteFileWithDocument(IndexWriter indexWriter, String fmUuid) { long current = 0; try { indexWriter.deleteDocuments(new Term("UUID", fmUuid)); current++; } catch (Exception e) { e.printStackTrace(); } return current; } /** * 批量删除索引 */ @Override public void deteleLuceneLs(List<FileManage> fmList) { for (FileManage fm : fmList) { this.deteleLucene(fm.getUuid()); } } /** * 每个档案创建一个Lucene的document(创建、添加、更新索引用到此方法) * @param indexWriter * @param current * @param fm * @return */ private Document initLuceneDocument(IndexWriter indexWriter, long current, FileManage fm) { String fileName = ""; String fileType = ""; String fileContent = ""; // TODO 版本号待定 List<EdocFileObjectRelation> fileList = docService.queryFiles(DocConstant.NO_DIR, fm.getUuid(), fm.getClass() .getName(), null); if (fileList.size() > 0) { for (EdocFileObjectRelation file : fileList) { InputStream fs = docService.getFileInputStream(file.getFileId()); fileName = file.getEdocFile().getName(); fileType = fileName.substring(fileName.lastIndexOf(".") + 1); fileContent += FileManageUtil.getContent(fileType, fs); // String content = fileContentService(file.getEdocFile().) } } /** * 写入索引文件 */ Document doc = new Document(); doc.add(new Field("UUID", fm.getUuid(), Store.YES, Index.NOT_ANALYZED)); // doc.add(new Field("FILENAME", fileName, Store.YES, // Index.NO));// 文档名称 if (!SzglCommonUtil.strIsNull(fileContent)) { doc.add(new Field("CONTENT", fileContent, Store.YES, Index.ANALYZED));// 具体内容 } if (!SzglCommonUtil.strIsNull(fm.getTitle())) { doc.add(new Field("TITLE", fm.getTitle(), Store.YES, Index.ANALYZED));// 档案标题 } if (!SzglCommonUtil.strIsNull(fm.getDocNum())) { doc.add(new Field("DOCNUM", fm.getDocNum(), Store.YES, Index.ANALYZED));// 档案文号 } if (fm.getFromMan() != null) { doc.add(new Field("FROMMAN", fm.getFromMan().toString(), Store.YES, Index.ANALYZED));// 创建人 } if (fm.getType() != null) { doc.add(new Field("TYPE", fm.getType(), Store.YES, Index.ANALYZED));// 档案类别 } if (fm.getStatus() != null) { doc.add(new Field("STATUS", fm.getStatus(), Store.YES, Index.ANALYZED));// 档案状态 } if (!SzglCommonUtil.strIsNull(fm.getIsShare())) { doc.add(new Field("ISHARE", fm.getIsShare(), Store.YES, Index.ANALYZED));// 档案是否共享 } if (fm.getFromTime() != null) { doc.add(new Field("FROMTIME", fm.getFromTime() + "", Store.YES, Index.ANALYZED));// 档案传来时间 } if (!SzglCommonUtil.strIsNull(fm.getBoxUuid())) { FileBox filebox = JPAUtil.loadById(FileBox.class, fm.getBoxUuid()); if (!SzglCommonUtil.strIsNull(filebox.getYearCode())) { doc.add(new Field("YEARCODE", filebox.getYearCode(), Store.YES, Index.ANALYZED));// 档案所属的档案盒年度 } } return doc; } private boolean delAllFile(File file) { boolean flag = false; if (file != null) { File[] tempList = file.listFiles(); File temp = null; for (int i = 0; i < tempList.length; i++) { temp = tempList[i]; if (temp.isFile()) { temp.delete(); } } } return flag; } /** * 显示时间 */ private void showtime() { long time1 = System.currentTimeMillis(); if (time > 0) { LogUtil.getAppLoger().debug("MilliSecond:" + (time1 - time)); } else { LogUtil.getAppLoger().debug("Start time:" + (new Timestamp(System.currentTimeMillis()))); } time = time1; } private void creatFile(File file) { if (!file.exists()) { file.mkdirs(); } } private long initFile(IndexWriter indexWriter) { long current = 0; String jpql = "select file from FileManage file where 1=1"; List<FileManage> fmLs = (List) JPAUtil.find(jpql); for (FileManage fm : fmLs) { // TODO 版本号待定 List<EdocFileObjectRelation> fileList = docService.queryFiles(DocConstant.NO_DIR, fm.getUuid(), fm .getClass().getName(), null); if (fileList.size() > 0) { for (EdocFileObjectRelation file : fileList) { InputStream fs = docService.getFileInputStream(file.getFileId()); String fileName = file.getEdocFile().getName(); String fileType = fileName.substring(fileName.lastIndexOf(".") + 1); String fileContent = FileManageUtil.getContent(fileType, fs); // String content = fileContentService(file.getEdocFile().) /** * 写入索引文件 */ try { Document doc = new Document(); // doc.add(new Field("TYPE", fm.getType(), Store.YES, // Index.ANALYZED)); // 档案类别 doc.add(new Field("UUID", fm.getUuid(), Store.YES, Index.NO)); doc.add(new Field("FILENAME", fileName, Store.YES, Index.NO));// 文档名称 if (!SzglCommonUtil.strIsNull(fileContent)) { doc.add(new Field("CONTENT", fileContent, Store.YES, Index.ANALYZED));// 具体内容 } if (!SzglCommonUtil.strIsNull(fm.getTitle())) { doc.add(new Field("TITLE", fm.getTitle(), Store.YES, Index.ANALYZED));// 档案标题 } if (!SzglCommonUtil.strIsNull(fm.getDocNum())) { doc.add(new Field("DOCNUM", fm.getDocNum(), Store.YES, Index.ANALYZED));// 档案文号 } if (fm.getFromMan() != null) { doc.add(new Field("FROMMAN", fm.getFromMan().toString(), Store.YES, Index.ANALYZED));// 创建人 } indexWriter.addDocument(doc); current++; if ((current - (current / 10000) * 10000) == 0) { LogUtil.getAppLoger().debug("current row num:" + current); } } catch (Exception ex) { ex.printStackTrace(); } } } } return current; } }
根据索引去查询,并将关键词标红
package com.haiyisoft.szgl.file.service.impl; import java.io.File; /** * 通过Lucene查询 * * @author haojiahong * * <p>Modification History:</p> * <p>Date Author Description</p> * <p>--------------------------------------------------------------</p> * <p>20151102 haojiahong new</p> * <p> </p> */ @Component("fileSchByLuceneService") public class FileSchByLuceneServiceImpl implements FileSchByLuceneService { @Override public List<FileManage> retrieveByLucene(String keyword, String titleSch, String docNumSch, String typeSch, String isShareSch, Timestamp yearCodeBegin, Timestamp yearCodeEnd, SortParamList sortParamList, PageInfo pageInfo) { List<FileManage> result = new ArrayList<FileManage>(); IndexSearcher searcher = null; String indexDir = FileManage.searchCenterForDocPath; File file = new File(indexDir); if ((SzglCommonUtil.strIsNull(keyword) && SzglCommonUtil.strIsNull(docNumSch) && SzglCommonUtil.strIsNull(titleSch) && SzglCommonUtil.strIsNull(typeSch) && SzglCommonUtil .strIsNull(isShareSch)) || (!file.exists())) { if (pageInfo != null) { pageInfo.setAllRowNum(0); } return null; } try { Directory dir = FSDirectory.open(new File(indexDir)); IndexReader reader = IndexReader.open(dir); searcher = new IndexSearcher(reader); BooleanQuery query = new BooleanQuery(); Analyzer anal = new IKAnalyzer(); QueryParser qp = new QueryParser(Version.LUCENE_36, "CONTENT", anal); QueryParser qpTitle = new QueryParser(Version.LUCENE_36, "TITLE", anal); QueryParser qpDocNum = new QueryParser(Version.LUCENE_36, "DOCNUM", anal); QueryParser qpIshare = new QueryParser(Version.LUCENE_36, "ISHARE", anal); if (!SzglCommonUtil.strIsNull(keyword)) { query.add(qp.parse(keyword), Occur.MUST); } if (!SzglCommonUtil.strIsNull(titleSch)) { query.add(qpTitle.parse(titleSch), Occur.MUST); } if (!SzglCommonUtil.strIsNull(docNumSch)) { query.add(qpDocNum.parse(docNumSch), Occur.MUST); } if (!SzglCommonUtil.strIsNull(typeSch)) { query.add(new TermQuery(new Term("TYPE", typeSch)), Occur.MUST); } if (!SzglCommonUtil.strIsNull(isShareSch)) { query.add(qpIshare.parse(isShareSch), Occur.MUST); } if (yearCodeBegin != null || yearCodeEnd != null) { query.add(new TermRangeQuery("YEARCODE", yearCodeBegin.toString(), yearCodeEnd.toString(), true, true), Occur.MUST); } ScoreDoc[] hits = searcher.search(query, Integer.MAX_VALUE).scoreDocs; int nowPagestart = (pageInfo.getCurPageNum() - 1) * pageInfo.getRowOfPage();// 当前页第一条数据是总数的第几条数据 int allPage = hits.length;// 总条数 pageInfo.setAllRowNum(allPage); int nowPageEnd = (nowPagestart + pageInfo.getRowOfPage()) < allPage ? (nowPagestart + pageInfo .getRowOfPage()) : allPage; for (int i = nowPagestart; i < nowPageEnd; i++) { FileManage fm = new FileManage(); Document doc = searcher.doc(hits[i].doc); fm.setUuid(doc.get("UUID")); if (!SzglCommonUtil.strIsNull(docNumSch)) { fm.setDocNum(lighterStr(anal, qpDocNum.parse(docNumSch), doc.get("DOCNUM"), "DOCNUM")); } else { fm.setDocNum(doc.get("DOCNUM")); } if (!SzglCommonUtil.strIsNull(titleSch)) { fm.setTitle(lighterStr(anal, qpTitle.parse(titleSch), doc.get("TITLE"), "TITLE")); } else { fm.setTitle(doc.get("TITLE")); } if (!SzglCommonUtil.strIsNull(keyword)) { fm.setFileContent(lighterStr(anal, qp.parse(keyword), doc.get("CONTENT"), "CONTENT")); } else { fm.setFileContent(doc.get("CONTENT")); } fm.setFromMan(Long.valueOf(doc.get("FROMMAN"))); fm.setFileName(doc.get("FILENAME")); fm.setType(doc.get("TYPE")); fm.setStatus(doc.get("STATUS")); fm.setFromTime(Timestamp.valueOf(doc.get("FROMTIME"))); result.add(fm); } } catch (IOException e) { e.printStackTrace(); } catch (Exception e) { e.printStackTrace(); } finally { if (searcher != null) try { searcher.close(); } catch (IOException e) { e.printStackTrace(); } } return result; } private String lighterStr(Analyzer a, Query query, String txt, String fieldname) throws Exception { String str = null; QueryScorer scorer = new QueryScorer(query); Fragmenter fragmenter = new SimpleSpanFragmenter(scorer); Formatter fmt = new SimpleHTMLFormatter("<span style=‘color:red‘>", "</span>"); Highlighter lighter = new Highlighter(fmt, scorer); lighter.setTextFragmenter(fragmenter); str = lighter.getBestFragment(a, fieldname, txt); if (str == null) return txt; return str; } }
一些工具类,用于读文档内容。
* Copyright (C) 2014-2020 Yantai HaiYi Software Co.,Ltd package com.haiyisoft.szgl.file.util; import java.io.BufferedReader; /** * 档案管理工具类 * * @author haojiahong * * <p>Modification History:</p> * <p>Date Author Description</p> * <p>--------------------------------------------------------------</p> * <p>20151102 haojiahong new</p> * <p> </p> */ public class FileManageUtil { /** * 根据文件类型,获取文本内容 * @param type * @param fs * @return */ public static String getContent(String type, InputStream fs) { String text = null; if ("doc".equals(type)) { POITextExtractor ex = null; try { ex = new WordExtractor(fs); text = ex.getText(); } catch (Exception e) { e.printStackTrace(); } } else if ("docx".equals(type)) { POITextExtractor ex = null; try { OPCPackage opcPackage = OPCPackage.open(fs); ex = new XWPFWordExtractor(opcPackage); text = ex.getText(); } catch (Exception e) { e.printStackTrace(); } } else if ("txt".equals(type)) { BufferedReader reader = new BufferedReader(new InputStreamReader(fs)); String line = null; try { while ((line = reader.readLine()) != null) { text += line; } } catch (IOException e) { e.printStackTrace(); } } else if ("pdf".equals(type)) { try { PDDocument pdfDocument = PDDocument.load(fs); text = new PDFTextStripper().getText(pdfDocument); } catch (IOException e) { e.printStackTrace(); } } return text; } }
时间: 2024-10-17 23:58:51