diff --git a/imixs-workflow-engine/src/main/java/org/imixs/workflow/engine/DocumentService.java b/imixs-workflow-engine/src/main/java/org/imixs/workflow/engine/DocumentService.java index e5885b071..cc562f615 100644 --- a/imixs-workflow-engine/src/main/java/org/imixs/workflow/engine/DocumentService.java +++ b/imixs-workflow-engine/src/main/java/org/imixs/workflow/engine/DocumentService.java @@ -40,12 +40,12 @@ import java.util.Map; import java.util.StringTokenizer; import java.util.Vector; +import java.util.logging.Level; import java.util.logging.Logger; import javax.annotation.Resource; import javax.annotation.security.DeclareRoles; import javax.annotation.security.RolesAllowed; -import javax.ejb.EJB; import javax.ejb.LocalBean; import javax.ejb.SessionContext; import javax.ejb.Stateless; @@ -61,11 +61,12 @@ import org.imixs.workflow.ItemCollection; import org.imixs.workflow.WorkflowKernel; import org.imixs.workflow.engine.index.SearchService; -import org.imixs.workflow.engine.index.UpdateService; import org.imixs.workflow.engine.index.SortOrder; +import org.imixs.workflow.engine.index.UpdateService; import org.imixs.workflow.engine.jpa.Document; import org.imixs.workflow.exceptions.AccessDeniedException; import org.imixs.workflow.exceptions.InvalidAccessException; +import org.imixs.workflow.exceptions.PluginException; import org.imixs.workflow.exceptions.QueryException; /** @@ -135,6 +136,10 @@ public class DocumentService { public static final String ACCESSLEVEL_MANAGERACCESS = "org.imixs.ACCESSLEVEL.MANAGERACCESS"; + public static final String EVENTLOG_TOPIC_INDEX_ADD = "index.add"; + public static final String EVENTLOG_TOPIC_INDEX_REMOVE = "index.remove"; + + public static final String READACCESS = "$readaccess"; public static final String WRITEACCESS = "$writeaccess"; public static final String ISAUTHOR = "$isAuthor"; @@ -166,8 +171,11 @@ public class DocumentService { private UpdateService indexUpdateService; @Inject - private SearchService luceneSearchService; + private SearchService indexSearchService; + @Inject + private EventLogService eventLogService; + @Inject protected Event documentEvents; @@ -473,10 +481,10 @@ public ItemCollection save(ItemCollection document) throws AccessDeniedException // add/update document into lucene index if (!document.getItemValueBoolean(NOINDEX)) { - indexUpdateService.updateDocument(document); + addDocumentToIndex(document); } else { // remove from index - indexUpdateService.removeDocument(document.getUniqueID()); + removeDocumentFromIndex(document.getUniqueID()); } /* @@ -490,6 +498,50 @@ public ItemCollection save(ItemCollection document) throws AccessDeniedException // return the updated document return document; } + + + + /** + * This method adds a single document into the to the Lucene index. Before the + * document is added to the index, a new eventLog is created. The document will + * be indexed after the method flushEventLog is called. This method is called by + * the LuceneSearchService finder methods. + *

+ * The method supports committed read. This means that a running transaction + * will not read an uncommitted document from the Lucene index. + * + * + * @param documentContext + */ + public void addDocumentToIndex(ItemCollection document) { + // skip if the flag 'noindex' = true + if (!document.getItemValueBoolean(DocumentService.NOINDEX)) { + // write a new EventLog entry for each document.... + eventLogService.createEvent(EVENTLOG_TOPIC_INDEX_ADD, document.getUniqueID()); + } + } + + /** + * This method adds a new eventLog for a document to be deleted from the index. + * The document will be removed from the index after the method fluschEventLog + * is called. This method is called by the LuceneSearchService finder method + * only. + * + * + * @param uniqueID + * of the workitem to be removed + * @throws PluginException + */ + public void removeDocumentFromIndex(String uniqueID) { + + long ltime = System.currentTimeMillis(); + eventLogService.createEvent(EVENTLOG_TOPIC_INDEX_REMOVE, uniqueID); + if (logger.isLoggable(Level.FINE)) { + logger.fine("... update eventLog cache in " + (System.currentTimeMillis() - ltime) + + " ms (1 document to be removed)"); + } + } + /** * This method saves a workitem in a new transaction. The method can be used by @@ -619,7 +671,7 @@ public void remove(ItemCollection document) throws AccessDeniedException { manager.remove(persistedDocument); // remove document form index - @see issue #412 if (!document.getItemValueBoolean(NOINDEX)) { - indexUpdateService.removeDocument(document.getUniqueID()); + removeDocumentFromIndex(document.getUniqueID()); } } else @@ -665,7 +717,8 @@ public int count(String searchTerm) throws QueryException { * in case the searchterm is not understandable. */ public int count(String sSearchTerm, int maxResult) throws QueryException { - return luceneSearchService.getTotalHits(sSearchTerm, maxResult, null); + indexUpdateService.updateIndex(); + return indexSearchService.getTotalHits(sSearchTerm, maxResult, null); } /** @@ -749,7 +802,10 @@ public List find(String searchTerm, int pageSize, int pageIndex, sortOrder = new SortOrder(sortBy, sortReverse); } - return luceneSearchService.search(searchTerm, pageSize, pageIndex, sortOrder, null); + // flush eventlog (see issue #411) + indexUpdateService.updateIndex(); + + return indexSearchService.search(searchTerm, pageSize, pageIndex, sortOrder, null,false); } @@ -796,8 +852,12 @@ public List findStubs(String searchTerm, int pageSize, int pageI // it would be possible if we use a SortedSetSortField class here sortOrder = new SortOrder(sortBy,sortReverse); } + + // flush eventlog (see issue #411) + indexUpdateService.updateIndex(); + // find stubs only! - return luceneSearchService.search(searchTerm, pageSize, pageIndex, sortOrder, null, true); + return indexSearchService.search(searchTerm, pageSize, pageIndex, sortOrder, null, true); } diff --git a/imixs-workflow-engine/src/main/java/org/imixs/workflow/engine/adminp/AdminPService.java b/imixs-workflow-engine/src/main/java/org/imixs/workflow/engine/adminp/AdminPService.java index 9cdfe8491..0df421271 100644 --- a/imixs-workflow-engine/src/main/java/org/imixs/workflow/engine/adminp/AdminPService.java +++ b/imixs-workflow-engine/src/main/java/org/imixs/workflow/engine/adminp/AdminPService.java @@ -29,7 +29,6 @@ import javax.annotation.Resource; import javax.annotation.security.DeclareRoles; import javax.annotation.security.RunAs; -import javax.ejb.EJB; import javax.ejb.LocalBean; import javax.ejb.SessionContext; import javax.ejb.Stateless; @@ -81,7 +80,7 @@ public class AdminPService { public static final String JOB_RENAME_USER = "RENAME_USER"; - public static final String JOB_REBUILD_LUCENE_INDEX = "REBUILD_LUCENE_INDEX"; + public static final String JOB_REBUILD_INDEX ="JOB_REBUILD_INDEX"; public static final String JOB_UPGRADE = "UPGRADE"; public static final String JOB_MIGRATION = "MIGRATION"; private static final int DEFAULT_INTERVAL = 1; @@ -101,6 +100,8 @@ public class AdminPService { @Inject JobHandlerRenameUser jobHandlerRenameUser; + @Inject + JobHandlerRebuildIndex jobHandlerRebuildIndex; @Inject @Any @@ -226,6 +227,10 @@ public void scheduleTimer(javax.ejb.Timer timer) { if (job.equals(JOB_UPGRADE)) { jobHandler = jobHandlerUpgradeWorkitems; } + + if (job.equals(JOB_REBUILD_INDEX) || job.equals("REBUILD_LUCENE_INDEX")) { + jobHandler = jobHandlerRebuildIndex; + } if (jobHandler == null) { // try to find the jobHandler by CDI ..... diff --git a/imixs-workflow-lucene/src/main/java/org/imixs/workflow/engine/adminp/JobHandlerRebuildIndex.java b/imixs-workflow-engine/src/main/java/org/imixs/workflow/engine/adminp/JobHandlerRebuildIndex.java similarity index 92% rename from imixs-workflow-lucene/src/main/java/org/imixs/workflow/engine/adminp/JobHandlerRebuildIndex.java rename to imixs-workflow-engine/src/main/java/org/imixs/workflow/engine/adminp/JobHandlerRebuildIndex.java index 8cb37b8b5..5807b696b 100644 --- a/imixs-workflow-lucene/src/main/java/org/imixs/workflow/engine/adminp/JobHandlerRebuildIndex.java +++ b/imixs-workflow-engine/src/main/java/org/imixs/workflow/engine/adminp/JobHandlerRebuildIndex.java @@ -20,8 +20,8 @@ import org.eclipse.microprofile.config.inject.ConfigProperty; import org.imixs.workflow.ItemCollection; +import org.imixs.workflow.engine.index.UpdateService; import org.imixs.workflow.engine.jpa.Document; -import org.imixs.workflow.engine.lucene.LuceneIndexService; import org.imixs.workflow.exceptions.AccessDeniedException; import org.imixs.workflow.exceptions.InvalidAccessException; import org.imixs.workflow.exceptions.PluginException; @@ -62,7 +62,7 @@ public class JobHandlerRebuildIndex implements JobHandler { private EntityManager manager; @Inject - LuceneIndexService luceneIndexService; + UpdateService updateService; private static Logger logger = Logger.getLogger(JobHandlerRebuildIndex.class.getName()); @@ -91,9 +91,9 @@ public ItemCollection run(ItemCollection adminp) throws AdminPException { int blockCount = 0; // read blocksize and timeout.... - logger.info("...Job " + AdminPService.JOB_REBUILD_LUCENE_INDEX + " (" + adminp.getUniqueID() + logger.info("...Job " + AdminPService.JOB_REBUILD_INDEX + " (" + adminp.getUniqueID() + ") - lucene.rebuild.block_size=" + block_size); - logger.info("...Job " + AdminPService.JOB_REBUILD_LUCENE_INDEX + " (" + adminp.getUniqueID() + logger.info("...Job " + AdminPService.JOB_REBUILD_INDEX + " (" + adminp.getUniqueID() + ") - lucene.rebuild.time_out=" + time_out); try { @@ -116,7 +116,7 @@ public ItemCollection run(ItemCollection adminp) throws AdminPException { } // update the index - luceneIndexService.updateDocumentsUncommitted(resultList); + updateService.updateIndex(resultList); manager.flush(); // update count @@ -127,7 +127,7 @@ public ItemCollection run(ItemCollection adminp) throws AdminPException { if (time == 0) { time = 1; } - logger.info("...Job " + AdminPService.JOB_REBUILD_LUCENE_INDEX + " (" + adminp.getUniqueID() + logger.info("...Job " + AdminPService.JOB_REBUILD_INDEX + " (" + adminp.getUniqueID() + ") - ..." + totalCount + " documents indexed in " + time + " sec. ... "); blockCount = 0; } @@ -143,7 +143,7 @@ public ItemCollection run(ItemCollection adminp) throws AdminPException { time = 1; } if (time > time_out) { // suspend after 2 mintues (default 120).... - logger.info("...Job " + AdminPService.JOB_REBUILD_LUCENE_INDEX + " (" + adminp.getUniqueID() + logger.info("...Job " + AdminPService.JOB_REBUILD_INDEX + " (" + adminp.getUniqueID() + ") - suspended: " + totalCount + " documents indexed in " + time + " sec. "); adminp.replaceItemValue("_syncpoint", syncPoint); @@ -156,7 +156,7 @@ public ItemCollection run(ItemCollection adminp) throws AdminPException { } } catch (Exception e) { // print exception and stop job - logger.severe("...Job " + AdminPService.JOB_REBUILD_LUCENE_INDEX + " (" + adminp.getUniqueID() + logger.severe("...Job " + AdminPService.JOB_REBUILD_INDEX + " (" + adminp.getUniqueID() + ") - failed - " + e.getMessage() + " last syncpoint " + syncPoint + " - " + totalCount + " documents reindexed...."); e.printStackTrace(); @@ -177,7 +177,7 @@ public ItemCollection run(ItemCollection adminp) throws AdminPException { if (time == 0) { time = 1; } - logger.info("...Job " + AdminPService.JOB_REBUILD_LUCENE_INDEX + " (" + adminp.getUniqueID() + ") - Finished: " + logger.info("...Job " + AdminPService.JOB_REBUILD_INDEX + " (" + adminp.getUniqueID() + ") - Finished: " + totalCount + " documents indexed in " + time + " sec. "); adminp.replaceItemValue(JobHandler.ISCOMPLETED, true); diff --git a/imixs-workflow-engine/src/main/java/org/imixs/workflow/engine/index/SearchService.java b/imixs-workflow-engine/src/main/java/org/imixs/workflow/engine/index/SearchService.java index 37ec48e07..01c0920cc 100644 --- a/imixs-workflow-engine/src/main/java/org/imixs/workflow/engine/index/SearchService.java +++ b/imixs-workflow-engine/src/main/java/org/imixs/workflow/engine/index/SearchService.java @@ -58,7 +58,7 @@ public interface SearchService { * @return collection of search result * @throws QueryException */ - public List search(String sSearchTerm) throws QueryException; + //public List search(String sSearchTerm) throws QueryException; /** * Returns a collection of documents matching the provided search term. The @@ -73,7 +73,7 @@ public interface SearchService { * @return collection of search result * @throws QueryException */ - public List search(String sSearchTerm, int pageSize, int pageIndex) throws QueryException; + //public List search(String sSearchTerm, int pageSize, int pageIndex) throws QueryException; /** * Returns a collection of documents matching the provided search term. The term @@ -88,8 +88,8 @@ public interface SearchService { *

* */ - public List search(String sSearchTerm, int pageSize, int pageIndex, SortOrder sortOrder, - DefaultOperator defaultOperator) throws QueryException; +// public List search(String sSearchTerm, int pageSize, int pageIndex, SortOrder sortOrder, +// DefaultOperator defaultOperator) throws QueryException; /** * Returns a collection of documents matching the provided search term. The term diff --git a/imixs-workflow-engine/src/main/java/org/imixs/workflow/engine/index/UpdateService.java b/imixs-workflow-engine/src/main/java/org/imixs/workflow/engine/index/UpdateService.java index c5da2d7e2..0acf461ea 100644 --- a/imixs-workflow-engine/src/main/java/org/imixs/workflow/engine/index/UpdateService.java +++ b/imixs-workflow-engine/src/main/java/org/imixs/workflow/engine/index/UpdateService.java @@ -28,19 +28,23 @@ package org.imixs.workflow.engine.index; import java.util.Arrays; -import java.util.Collection; import java.util.List; import javax.ejb.Local; import org.imixs.workflow.ItemCollection; import org.imixs.workflow.exceptions.IndexException; -import org.imixs.workflow.exceptions.PluginException; /** - * The UpdateService provides methods to write Imixs Workitems into a search - * index. An ItemCollection can be added into the index by calling themethod - * updateDocument() + * The UpdateService defines methods to update the search index. These methods + * are called by the DocuentService. + *

+ * The method updateIndex(documents) writes documents immediately into + * the index. + *

+ * The method updateIndex() updates the search index based on the eventLog. + *

+ * The UpdateService provides also the default index schema. * * @see SchemaService * @version 1.0 @@ -61,46 +65,26 @@ public interface UpdateService { "$lasteventdate", "$creator", "$editor", "$lasteditor", "$owner", "namowner"); /** - * This method adds a single document into the to the Lucene index. Before the - * document is added to the index, a new eventLog is created. The document will - * be indexed after the method flushEventLog is called. This method is called by - * the LuceneSearchService finder methods. + * This method adds a collection of documents to the index. The documents are + * added immediately to the index. Calling this method within a running + * transaction leads to a uncommitted reads in the index. For transaction + * control, it is recommended to use instead the the method + * documentService.addDocumentToIndex() which takes care of uncommitted reads. *

- * The method supports committed read. This means that a running transaction - * will not read an uncommitted document from the Lucene index. + * This method is used by the JobHandlerRebuildIndex only. * - * - * @param documentContext - */ - public void updateDocument(ItemCollection documentContext); - - /** - * This method adds a collection of documents to the Lucene index. For each - * document in a given selection a new eventLog is created. The documents will - * be indexed after the method flushEventLog is called. This method is called by - * the LuceneSearchService finder methods. - *

- * The method supports committed read. This means that a running transaction - * will not read uncommitted documents from the Lucene index. - * - * @see updateDocumentsUncommitted * @param documents - * to be indexed + * of ItemCollections to be indexed * @throws IndexException */ - public void updateDocuments(Collection documents); + public void updateIndex(List documents); /** - * This method adds a new eventLog for a document to be deleted from the index. - * The document will be removed from the index after the method fluschEventLog - * is called. This method is called by the LuceneSearchService finder method - * only. + * This method updates the search index based on the eventLog. Documents are + * added by the DocumentService as events to the EventLogService. This ensures + * that only committed documents are added into the index. * - * - * @param uniqueID - * of the workitem to be removed - * @throws PluginException + * @see DocumentService */ - public void removeDocument(String uniqueID); - + public void updateIndex(); } diff --git a/imixs-workflow-lucene/src/main/java/org/imixs/workflow/engine/lucene/LuceneIndexService.java b/imixs-workflow-lucene/src/main/java/org/imixs/workflow/engine/lucene/LuceneIndexService.java index 04a390f7e..0ca633d2f 100644 --- a/imixs-workflow-lucene/src/main/java/org/imixs/workflow/engine/lucene/LuceneIndexService.java +++ b/imixs-workflow-lucene/src/main/java/org/imixs/workflow/engine/lucene/LuceneIndexService.java @@ -70,7 +70,6 @@ import org.imixs.workflow.engine.index.SchemaService; import org.imixs.workflow.engine.jpa.EventLog; import org.imixs.workflow.exceptions.IndexException; -import org.imixs.workflow.exceptions.PluginException; /** * This session ejb provides functionality to maintain a local Lucene index. @@ -82,8 +81,6 @@ public class LuceneIndexService { public static final int EVENTLOG_ENTRY_FLUSH_COUNT = 16; - public static final String EVENTLOG_TOPIC_ADD = "lucene.add"; - public static final String EVENTLOG_TOPIC_REMOVE = "lucene.remove"; public static final String ANONYMOUS = "ANONYMOUS"; public static final String DEFAULT_ANALYSER = "org.apache.lucene.analysis.standard.ClassicAnalyzer"; @@ -139,31 +136,11 @@ public void setLuceneAnalyserClass(String luceneAnalyserClass) { this.luceneAnalyserClass = luceneAnalyserClass; } - /** - * This method adds a new eventLog for a document to be deleted from the index. - * The document will be removed from the index after the method fluschEventLog - * is called. This method is called by the LuceneSearchService finder method - * only. - * - * - * @param uniqueID - * of the workitem to be removed - * @throws PluginException - */ - public void removeDocument(String uniqueID) { - - long ltime = System.currentTimeMillis(); - eventLogService.createEvent(EVENTLOG_TOPIC_REMOVE, uniqueID); - if (logger.isLoggable(Level.FINE)) { - logger.fine("... update eventLog cache in " + (System.currentTimeMillis() - ltime) - + " ms (1 document to be removed)"); - } - } - + /** - * Flush the EventLog cache. This method is called by the LuceneSerachServicen + * Flush the EventLog cache. This method is called by the LuceneSerachService * only. *

* The method flushes the cache in smaller blocks of the given junkSize. to @@ -232,7 +209,7 @@ private boolean flushEventLogByCount(int count) { long l = System.currentTimeMillis(); logger.finest("......flush eventlog cache...."); - List events = eventLogService.findEventsByTopic(count + 1, EVENTLOG_TOPIC_ADD, EVENTLOG_TOPIC_REMOVE); + List events = eventLogService.findEventsByTopic(count + 1, DocumentService.EVENTLOG_TOPIC_INDEX_ADD, DocumentService.EVENTLOG_TOPIC_INDEX_REMOVE); if (events != null && events.size() > 0) { try { @@ -246,7 +223,7 @@ private boolean flushEventLogByCount(int count) { // if the document was found we add/update the index. Otherwise we remove the // document form the index. - if (doc != null && EVENTLOG_TOPIC_ADD.equals(eventLogEntry.getTopic())) { + if (doc != null && DocumentService.EVENTLOG_TOPIC_INDEX_ADD.equals(eventLogEntry.getTopic())) { // add workitem to search index.... long l2 = System.currentTimeMillis(); ItemCollection workitem = new ItemCollection(); @@ -477,10 +454,41 @@ public void rebuildIndex(Directory indexDir) throws IOException { logger.info("...rebuild lucene index job created..."); ItemCollection job = new ItemCollection(); job.replaceItemValue("numinterval", 2); // 2 minutes - job.replaceItemValue("job", AdminPService.JOB_REBUILD_LUCENE_INDEX); + job.replaceItemValue("job", AdminPService.JOB_REBUILD_INDEX); adminPService.createJob(job); } + + /** + * This method creates a new instance of a lucene IndexWriter. + * + * The location of the lucene index in the filesystem is read from the + * imixs.properties + * + * @return + * @throws IOException + */ + public IndexWriter createIndexWriter() throws IOException { + // create a IndexWriter Instance + Directory indexDir = FSDirectory.open(Paths.get(luceneIndexDir)); + // verify existence of index directory... + if (!DirectoryReader.indexExists(indexDir)) { + logger.info("...lucene index does not yet exist, initialize the index now...."); + rebuildIndex(indexDir); + } + IndexWriterConfig indexWriterConfig; + // indexWriterConfig = new IndexWriterConfig(new ClassicAnalyzer()); + try { + // issue #429 + indexWriterConfig = new IndexWriterConfig((Analyzer) Class.forName(luceneAnalyserClass).newInstance()); + } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { + throw new IndexException(IndexException.INVALID_INDEX, + "Unable to create analyzer '" + luceneAnalyserClass + "'", e); + } + + return new IndexWriter(indexDir, indexWriterConfig); + } + /** * This method adds a collection of documents to the Lucene index. The documents * are added immediately to the index. Calling this method within a running @@ -533,38 +541,4 @@ public void updateDocumentsUncommitted(Collection documents) { + " workitems total)"); } } - - - - /** - * This method creates a new instance of a lucene IndexWriter. - * - * The location of the lucene index in the filesystem is read from the - * imixs.properties - * - * @return - * @throws IOException - */ - public IndexWriter createIndexWriter() throws IOException { - // create a IndexWriter Instance - Directory indexDir = FSDirectory.open(Paths.get(luceneIndexDir)); - // verify existence of index directory... - if (!DirectoryReader.indexExists(indexDir)) { - logger.info("...lucene index does not yet exist, initialize the index now...."); - rebuildIndex(indexDir); - } - IndexWriterConfig indexWriterConfig; - // indexWriterConfig = new IndexWriterConfig(new ClassicAnalyzer()); - try { - // issue #429 - indexWriterConfig = new IndexWriterConfig((Analyzer) Class.forName(luceneAnalyserClass).newInstance()); - } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { - throw new IndexException(IndexException.INVALID_INDEX, - "Unable to create analyzer '" + luceneAnalyserClass + "'", e); - } - - return new IndexWriter(indexDir, indexWriterConfig); - } - - } diff --git a/imixs-workflow-lucene/src/main/java/org/imixs/workflow/engine/lucene/LuceneSearchService.java b/imixs-workflow-lucene/src/main/java/org/imixs/workflow/engine/lucene/LuceneSearchService.java index 3b10fe933..0d2b04cc7 100644 --- a/imixs-workflow-lucene/src/main/java/org/imixs/workflow/engine/lucene/LuceneSearchService.java +++ b/imixs-workflow-lucene/src/main/java/org/imixs/workflow/engine/lucene/LuceneSearchService.java @@ -128,11 +128,11 @@ public class LuceneSearchService implements SearchService { * @return collection of search result * @throws QueryException */ - @Override - public List search(String sSearchTerm) throws QueryException { - // no sort order - return search(sSearchTerm, DEFAULT_MAX_SEARCH_RESULT, 0, null, DefaultOperator.AND); - } +// @Override +// public List search(String sSearchTerm) throws QueryException { +// // no sort order +// return search(sSearchTerm, DEFAULT_MAX_SEARCH_RESULT, 0, null, DefaultOperator.AND); +// } /** * Returns a collection of documents matching the provided search term. The @@ -147,12 +147,11 @@ public List search(String sSearchTerm) throws QueryException { * @return collection of search result * @throws QueryException */ - @Override - - public List search(String sSearchTerm, int pageSize, int pageIndex) throws QueryException { - // no sort order - return search(sSearchTerm, pageSize, pageIndex, null, null); - } +// @Override +// public List search(String sSearchTerm, int pageSize, int pageIndex) throws QueryException { +// // no sort order +// return search(sSearchTerm, pageSize, pageIndex, null, null); +// } /** * Returns a collection of documents matching the provided search term. The term @@ -167,12 +166,12 @@ public List search(String sSearchTerm, int pageSize, int pageInd *

* */ - @Override - public List search(String sSearchTerm, int pageSize, int pageIndex, - org.imixs.workflow.engine.index.SortOrder sortOrder, DefaultOperator defaultOperator) - throws QueryException { - return search(sSearchTerm, pageSize, pageIndex, sortOrder, defaultOperator, false); - } +// @Override +// public List search(String sSearchTerm, int pageSize, int pageIndex, +// org.imixs.workflow.engine.index.SortOrder sortOrder, DefaultOperator defaultOperator) +// throws QueryException { +// return search(sSearchTerm, pageSize, pageIndex, sortOrder, defaultOperator, false); +// } /** * Returns a collection of documents matching the provided search term. The term @@ -212,7 +211,7 @@ public List search(String searchTerm, int pageSize, int pageInde long ltime = System.currentTimeMillis(); // flush eventlog (see issue #411) - flush(); + //flush(); // see issue #382 /* @@ -321,7 +320,7 @@ public List search(String searchTerm, int pageSize, int pageInde workitems.add(imixsDoc); } else { logger.warning("lucene index returned unreadable workitem : " + sID); - luceneIndexService.removeDocument(sID); + documentService.removeDocumentFromIndex(sID); // this situation happens if the search index returned // documents the current user has no read access. // this should normally avoided with the $readaccess @@ -349,32 +348,6 @@ public List search(String searchTerm, int pageSize, int pageInde - private Sort buildLuceneSort(org.imixs.workflow.engine.index.SortOrder sortOrder) { - Sort sort = null; - // we do not support multi values here - see - // LuceneUpdateService.addItemValues - // it would be possible if we use a SortedSetSortField class here - sort = new Sort(new SortField[] { new SortField(sortOrder.getField(), SortField.Type.STRING, sortOrder.isReverse()) }); - return sort; - } - - - - /** - * This method flush the event log. - */ - public void flush() { - long ltime = System.currentTimeMillis(); - // flush eventlog (see issue #411) - int flushCount = 0; - while (luceneIndexService.flushEventLog(2048) == false) { - // repeat flush.... - flushCount = +2048; - logger.info("...flush event log: " + flushCount + " entries updated in " - + (System.currentTimeMillis() - ltime) + "ms ..."); - } - } - /** * Returns the total hits for a given search term from the lucene index. The * method did not load any data. The provided search term will we extended with @@ -398,46 +371,46 @@ public int getTotalHits(final String _searchTerm, final int _maxResult, final De throws QueryException { int result; int maxResult = _maxResult; - + if (maxResult <= 0) { maxResult = DEFAULT_MAX_SEARCH_RESULT; } - + String sSearchTerm =schemaService.getExtendedSearchTerm(_searchTerm); // test if searchtem is provided if (sSearchTerm == null || "".equals(sSearchTerm)) { return 0; } - + try { IndexSearcher searcher = createIndexSearcher(); QueryParser parser = createQueryParser(); - + parser.setAllowLeadingWildcard(true); - + // set default operator? if (defaultOperator.equals(Operator.AND)) { parser.setDefaultOperator(org.apache.lucene.queryparser.classic.QueryParser.Operator.AND); } else { parser.setDefaultOperator(org.apache.lucene.queryparser.classic.QueryParser.Operator.OR); - + } - + TopDocsCollector collector = null; - + Query query = parser.parse(sSearchTerm); // MAX_SEARCH_RESULT is limiting the total number of hits collector = TopScoreDocCollector.create(maxResult); - + // - ignore time limiting for now // Counter clock = Counter.newCounter(true); // TimeLimitingCollector timeLimitingCollector = new // TimeLimitingCollector(collector, clock, 10); - + // start search.... searcher.search(query, collector); result = collector.getTotalHits(); - + logger.finest("......lucene count result = " + result); } catch (IOException e) { // in case of an IOException we just print an error message and @@ -448,11 +421,12 @@ public int getTotalHits(final String _searchTerm, final int _maxResult, final De logger.severe("Lucene search error: " + e.getMessage()); throw new QueryException(QueryException.QUERY_NOT_UNDERSTANDABLE, e.getMessage(), e); } - + return result; } - + + /** * Creates a Lucene FSDirectory Instance. The method uses the proeprty * LockFactory to set a custom LockFactory. @@ -587,6 +561,17 @@ ItemCollection convertLuceneDocument(Document luceneDoc, SimpleDateFormat lucene return imixsDoc; } + private Sort buildLuceneSort(org.imixs.workflow.engine.index.SortOrder sortOrder) { + Sort sort = null; + // we do not support multi values here - see + // LuceneUpdateService.addItemValues + // it would be possible if we use a SortedSetSortField class here + sort = new Sort(new SortField[] { new SortField(sortOrder.getField(), SortField.Type.STRING, sortOrder.isReverse()) }); + return sort; + } + + + /** * Helper method to check for numbers. * diff --git a/imixs-workflow-lucene/src/main/java/org/imixs/workflow/engine/lucene/LuceneUpdateService.java b/imixs-workflow-lucene/src/main/java/org/imixs/workflow/engine/lucene/LuceneUpdateService.java index ae2cdb03e..a6493493f 100644 --- a/imixs-workflow-lucene/src/main/java/org/imixs/workflow/engine/lucene/LuceneUpdateService.java +++ b/imixs-workflow-lucene/src/main/java/org/imixs/workflow/engine/lucene/LuceneUpdateService.java @@ -27,10 +27,7 @@ package org.imixs.workflow.engine.lucene; -import java.util.ArrayList; -import java.util.Collection; import java.util.List; -import java.util.logging.Level; import java.util.logging.Logger; import javax.annotation.PostConstruct; @@ -38,11 +35,8 @@ import javax.inject.Inject; import org.imixs.workflow.ItemCollection; -import org.imixs.workflow.engine.DocumentService; -import org.imixs.workflow.engine.EventLogService; import org.imixs.workflow.engine.index.UpdateService; import org.imixs.workflow.exceptions.IndexException; -import org.imixs.workflow.exceptions.PluginException; /** * The LuceneUpdateService provides methods to write Imixs Workitems into a @@ -74,10 +68,6 @@ public class LuceneUpdateService implements UpdateService { - - @Inject - private EventLogService eventLogService; - @Inject private LuceneIndexService luceneIndexService; @@ -92,80 +82,46 @@ public class LuceneUpdateService implements UpdateService { */ @PostConstruct void init() { - logger.finest("......lucene IndexDir=" + luceneIndexService.getLuceneIndexDir()); - } /** - * This method adds a single document into the to the Lucene index. Before the - * document is added to the index, a new eventLog is created. The document will - * be indexed after the method flushEventLog is called. This method is called by - * the LuceneSearchService finder methods. + * This method adds a collection of documents to the Lucene index. The documents + * are added immediately to the index. Calling this method within a running + * transaction leads to a uncommitted reads in the index. For transaction + * control, it is recommended to use instead the the method updateDocumetns() + * which takes care of uncommitted reads. *

- * The method supports committed read. This means that a running transaction - * will not read an uncommitted document from the Lucene index. - * + * This method is used by the JobHandlerRebuildIndex only. * - * @param documentContext - */ - public void updateDocument(ItemCollection documentContext) { - // adds the document into a empty Collection and call the method - // updateDocuments. - List documents = new ArrayList(); - documents.add(documentContext); - updateDocuments(documents); - } - - /** - * This method adds a collection of documents to the Lucene index. For each - * document in a given selection a new eventLog is created. The documents will - * be indexed after the method flushEventLog is called. This method is called by - * the LuceneSearchService finder methods. - *

- * The method supports committed read. This means that a running transaction - * will not read uncommitted documents from the Lucene index. - * - * @see updateDocumentsUncommitted * @param documents - * to be indexed + * of ItemCollections to be indexed * @throws IndexException */ - public void updateDocuments(Collection documents) { - long ltime = System.currentTimeMillis(); - - // write a new EventLog entry for each document.... - for (ItemCollection workitem : documents) { - // skip if the flag 'noindex' = true - if (!workitem.getItemValueBoolean(DocumentService.NOINDEX)) { - eventLogService.createEvent(LuceneIndexService.EVENTLOG_TOPIC_ADD, workitem.getUniqueID()); - } - } - - if (logger.isLoggable(Level.FINE)) { - logger.fine("... update eventLog cache in " + (System.currentTimeMillis() - ltime) + " ms (" - + documents.size() + " documents to be index)"); - } + @Override + public void updateIndex(List documents) { + luceneIndexService.updateDocumentsUncommitted(documents); + } + /** - * This method adds a new eventLog for a document to be deleted from the index. - * The document will be removed from the index after the method fluschEventLog - * is called. This method is called by the LuceneSearchService finder method - * only. - * - * - * @param uniqueID - * of the workitem to be removed - * @throws PluginException + * This method flush the event log. */ - public void removeDocument(String uniqueID) { - luceneIndexService.removeDocument(uniqueID); - + @Override + public void updateIndex() { + long ltime = System.currentTimeMillis(); + // flush eventlog (see issue #411) + int flushCount = 0; + while (luceneIndexService.flushEventLog(2048) == false) { + // repeat flush.... + flushCount = +2048; + logger.info("...flush event log: " + flushCount + " entries updated in " + + (System.currentTimeMillis() - ltime) + "ms ..."); + } } - } diff --git a/imixs-workflow-solr/src/main/java/org/imixs/workflow/engine/solr/SolrIndexService.java b/imixs-workflow-solr/src/main/java/org/imixs/workflow/engine/solr/SolrIndexService.java index 85b03e8bf..3daf0d83a 100644 --- a/imixs-workflow-solr/src/main/java/org/imixs/workflow/engine/solr/SolrIndexService.java +++ b/imixs-workflow-solr/src/main/java/org/imixs/workflow/engine/solr/SolrIndexService.java @@ -158,6 +158,9 @@ public void updateSchema(String schema) throws RestAPIException { } } + + + /** * This method returns a JSON structure to to update an existing Solr schema. * The method adds all fields into a solr update definition that did not yet diff --git a/imixs-workflow-solr/src/main/java/org/imixs/workflow/engine/solr/SolrSearchService.java b/imixs-workflow-solr/src/main/java/org/imixs/workflow/engine/solr/SolrSearchService.java index 69aa065e9..256de0487 100644 --- a/imixs-workflow-solr/src/main/java/org/imixs/workflow/engine/solr/SolrSearchService.java +++ b/imixs-workflow-solr/src/main/java/org/imixs/workflow/engine/solr/SolrSearchService.java @@ -86,11 +86,11 @@ public class SolrSearchService implements SearchService { * @return collection of search result * @throws QueryException */ - @Override - public List search(String sSearchTerm) throws QueryException { - // no sort order - return search(sSearchTerm, DEFAULT_MAX_SEARCH_RESULT, 0, null, DefaultOperator.AND); - } +// @Override +// public List search(String sSearchTerm) throws QueryException { +// // no sort order +// return search(sSearchTerm, DEFAULT_MAX_SEARCH_RESULT, 0, null, DefaultOperator.AND); +// } /** * Returns a collection of documents matching the provided search term. The @@ -105,12 +105,11 @@ public List search(String sSearchTerm) throws QueryException { * @return collection of search result * @throws QueryException */ - @Override - - public List search(String sSearchTerm, int pageSize, int pageIndex) throws QueryException { - // no sort order - return search(sSearchTerm, pageSize, pageIndex, null, null); - } +// @Override +// public List search(String sSearchTerm, int pageSize, int pageIndex) throws QueryException { +// // no sort order +// return search(sSearchTerm, pageSize, pageIndex, null, null); +// } /** * Returns a collection of documents matching the provided search term. The term @@ -125,12 +124,12 @@ public List search(String sSearchTerm, int pageSize, int pageInd *

* */ - @Override - public List search(String sSearchTerm, int pageSize, int pageIndex, - org.imixs.workflow.engine.index.SortOrder sortOrder, DefaultOperator defaultOperator) - throws QueryException { - return search(sSearchTerm, pageSize, pageIndex, sortOrder, defaultOperator, false); - } +// @Override +// public List search(String sSearchTerm, int pageSize, int pageIndex, +// org.imixs.workflow.engine.index.SortOrder sortOrder, DefaultOperator defaultOperator) +// throws QueryException { +// return search(sSearchTerm, pageSize, pageIndex, sortOrder, defaultOperator, false); +// } /** * Returns a collection of documents matching the provided search term. The term diff --git a/imixs-workflow-solr/src/main/java/org/imixs/workflow/engine/solr/SolrUpdateService.java b/imixs-workflow-solr/src/main/java/org/imixs/workflow/engine/solr/SolrUpdateService.java index 4dc33b234..4dc5dadd7 100644 --- a/imixs-workflow-solr/src/main/java/org/imixs/workflow/engine/solr/SolrUpdateService.java +++ b/imixs-workflow-solr/src/main/java/org/imixs/workflow/engine/solr/SolrUpdateService.java @@ -28,6 +28,7 @@ package org.imixs.workflow.engine.solr; import java.util.Collection; +import java.util.List; import java.util.logging.Logger; import javax.annotation.PostConstruct; @@ -59,10 +60,7 @@ public class SolrUpdateService implements UpdateService { - - @Inject - private EventLogService eventLogService; - + private static Logger logger = Logger.getLogger(SolrUpdateService.class.getName()); /** @@ -79,58 +77,33 @@ void init() { } /** - * This method adds a single document into the to the Lucene index. Before the - * document is added to the index, a new eventLog is created. The document will - * be indexed after the method flushEventLog is called. This method is called by - * the LuceneSearchService finder methods. - *

- * The method supports committed read. This means that a running transaction - * will not read an uncommitted document from the Lucene index. - * - * - * @param documentContext - */ - public void updateDocument(ItemCollection documentContext) { - logger.info("...TBD"); - } - - /** - * This method adds a collection of documents to the Lucene index. For each - * document in a given selection a new eventLog is created. The documents will - * be indexed after the method flushEventLog is called. This method is called by - * the LuceneSearchService finder methods. + * This method adds a collection of documents to the Lucene index. The documents + * are added immediately to the index. Calling this method within a running + * transaction leads to a uncommitted reads in the index. For transaction + * control, it is recommended to use instead the the method updateDocumetns() + * which takes care of uncommitted reads. *

- * The method supports committed read. This means that a running transaction - * will not read uncommitted documents from the Lucene index. + * This method is used by the JobHandlerRebuildIndex only. * - * @see updateDocumentsUncommitted * @param documents - * to be indexed + * of ItemCollections to be indexed * @throws IndexException */ - public void updateDocuments(Collection documents) { - logger.info("...TBD"); + @Override + public void updateIndex(List documents) { + logger.warning(" unimplemented !!!!"); } - - /** - * This method adds a new eventLog for a document to be deleted from the index. - * The document will be removed from the index after the method fluschEventLog - * is called. This method is called by the LuceneSearchService finder method - * only. - * - * - * @param uniqueID - * of the workitem to be removed - * @throws PluginException - */ - public void removeDocument(String uniqueID) { - logger.info("...TBD"); + + @Override + public void updateIndex() { - + logger.warning(" unimplemented !!!!"); } + +