From 9040f4781926727808661e24980ea1d7ac411e74 Mon Sep 17 00:00:00 2001 From: Ralph Soika Date: Sun, 1 Sep 2019 00:28:19 +0200 Subject: [PATCH] implementation search method Issue #554 --- .../org/imixs/workflow/util/JSONParser.java | 3 + .../imixs/workflow/engine/SetupService.java | 17 +- .../engine/lucene/LuceneIndexService.java | 6 +- .../engine/lucene/LuceneUpdateService.java | 15 +- .../engine/solr/SolrIndexService.java | 410 +++++++++++++++--- .../engine/solr/SolrSearchService.java | 122 +++--- .../engine/solr/SolrUpdateService.java | 27 +- .../engine/solr/TestParseSolrJSONResult.java | 73 ++++ 8 files changed, 516 insertions(+), 157 deletions(-) create mode 100644 imixs-workflow-index-solr/src/test/java/org/imixs/workflow/engine/solr/TestParseSolrJSONResult.java diff --git a/imixs-workflow-core/src/main/java/org/imixs/workflow/util/JSONParser.java b/imixs-workflow-core/src/main/java/org/imixs/workflow/util/JSONParser.java index 150ef93de..78571b507 100644 --- a/imixs-workflow-core/src/main/java/org/imixs/workflow/util/JSONParser.java +++ b/imixs-workflow-core/src/main/java/org/imixs/workflow/util/JSONParser.java @@ -45,6 +45,9 @@ public class JSONParser { * @return - the json value or the json object for the corresponding json key */ public static String getKey(String key, String json) { + if (json==null || json.isEmpty()) { + return null; + } String result = null; // now extract the key JsonParser parser = Json.createParser(new StringReader(json)); diff --git a/imixs-workflow-engine/src/main/java/org/imixs/workflow/engine/SetupService.java b/imixs-workflow-engine/src/main/java/org/imixs/workflow/engine/SetupService.java index a07e2ad16..91b5cda9e 100644 --- a/imixs-workflow-engine/src/main/java/org/imixs/workflow/engine/SetupService.java +++ b/imixs-workflow-engine/src/main/java/org/imixs/workflow/engine/SetupService.java @@ -38,7 +38,6 @@ import javax.annotation.Resource; import javax.annotation.security.DeclareRoles; import javax.annotation.security.RunAs; -import javax.ejb.EJB; import javax.ejb.Singleton; import javax.ejb.Startup; import javax.ejb.Timer; @@ -154,12 +153,6 @@ public void startup() { } } - // migrate old workflow scheduler - migrateWorkflowScheduler(); - - // next start optional schedulers - logger.info("...initalizing schedulers..."); - schedulerService.startAllSchedulers(); // Finally fire the SetupEvent. This allows CDI Observers to react on the setup if (setupEvents != null) { @@ -170,6 +163,16 @@ public void startup() { logger.warning("Missing CDI support for Event !"); } + + + + // migrate old workflow scheduler + migrateWorkflowScheduler(); + + // Finally start optional schedulers + logger.info("...initalizing schedulers..."); + schedulerService.startAllSchedulers(); + } /** diff --git a/imixs-workflow-index-lucene/src/main/java/org/imixs/workflow/engine/lucene/LuceneIndexService.java b/imixs-workflow-index-lucene/src/main/java/org/imixs/workflow/engine/lucene/LuceneIndexService.java index 83ef13fbf..df7fd224a 100644 --- a/imixs-workflow-index-lucene/src/main/java/org/imixs/workflow/engine/lucene/LuceneIndexService.java +++ b/imixs-workflow-index-lucene/src/main/java/org/imixs/workflow/engine/lucene/LuceneIndexService.java @@ -84,9 +84,7 @@ public class LuceneIndexService { public static final String ANONYMOUS = "ANONYMOUS"; public static final String DEFAULT_ANALYSER = "org.apache.lucene.analysis.standard.ClassicAnalyzer"; - public static final String DEFAULT_INDEX_DIRECTORY = "imixs-workflow-index"; - - + public static final String DEFAULT_INDEX_DIRECTORY = "imixs-workflow-index"; @PersistenceContext(unitName = "org.imixs.workflow.jpa") private EntityManager manager; @@ -228,7 +226,7 @@ public void rebuildIndex(Directory indexDir) throws IOException { * of ItemCollections to be indexed * @throws IndexException */ - public void updateDocumentsUncommitted(Collection documents) { + public void indexDocuments(Collection documents) { IndexWriter awriter = null; long ltime = System.currentTimeMillis(); diff --git a/imixs-workflow-index-lucene/src/main/java/org/imixs/workflow/engine/lucene/LuceneUpdateService.java b/imixs-workflow-index-lucene/src/main/java/org/imixs/workflow/engine/lucene/LuceneUpdateService.java index a6493493f..d622ebfff 100644 --- a/imixs-workflow-index-lucene/src/main/java/org/imixs/workflow/engine/lucene/LuceneUpdateService.java +++ b/imixs-workflow-index-lucene/src/main/java/org/imixs/workflow/engine/lucene/LuceneUpdateService.java @@ -30,7 +30,6 @@ import java.util.List; import java.util.logging.Logger; -import javax.annotation.PostConstruct; import javax.ejb.Singleton; import javax.inject.Inject; @@ -70,21 +69,9 @@ public class LuceneUpdateService implements UpdateService { @Inject private LuceneIndexService luceneIndexService; - private static Logger logger = Logger.getLogger(LuceneUpdateService.class.getName()); - /** - * PostContruct event - The method loads the lucene index properties from the - * imixs.properties file from the classpath. If no properties are defined the - * method terminates. - * - */ - @PostConstruct - void init() { - logger.finest("......lucene IndexDir=" + luceneIndexService.getLuceneIndexDir()); - } - /** * This method adds a collection of documents to the Lucene index. The documents * are added immediately to the index. Calling this method within a running @@ -100,7 +87,7 @@ void init() { */ @Override public void updateIndex(List documents) { - luceneIndexService.updateDocumentsUncommitted(documents); + luceneIndexService.indexDocuments(documents); } diff --git a/imixs-workflow-index-solr/src/main/java/org/imixs/workflow/engine/solr/SolrIndexService.java b/imixs-workflow-index-solr/src/main/java/org/imixs/workflow/engine/solr/SolrIndexService.java index 4b45ebd3f..93758782d 100644 --- a/imixs-workflow-index-solr/src/main/java/org/imixs/workflow/engine/solr/SolrIndexService.java +++ b/imixs-workflow-index-solr/src/main/java/org/imixs/workflow/engine/solr/SolrIndexService.java @@ -27,8 +27,10 @@ package org.imixs.workflow.engine.solr; -import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Calendar; import java.util.Date; import java.util.List; @@ -39,15 +41,23 @@ import javax.annotation.security.DeclareRoles; import javax.annotation.security.RolesAllowed; import javax.ejb.Stateless; +import javax.ejb.TransactionAttribute; +import javax.ejb.TransactionAttributeType; import javax.enterprise.event.Observes; import javax.inject.Inject; +import javax.persistence.EntityManager; +import javax.persistence.PersistenceContext; import org.eclipse.microprofile.config.inject.ConfigProperty; import org.imixs.workflow.ItemCollection; +import org.imixs.workflow.engine.DocumentService; +import org.imixs.workflow.engine.EventLogService; import org.imixs.workflow.engine.SetupEvent; import org.imixs.workflow.engine.adminp.AdminPService; import org.imixs.workflow.engine.index.SchemaService; +import org.imixs.workflow.engine.jpa.EventLog; import org.imixs.workflow.exceptions.IndexException; +import org.imixs.workflow.exceptions.QueryException; import org.imixs.workflow.services.rest.BasicAuthenticator; import org.imixs.workflow.services.rest.RestAPIException; import org.imixs.workflow.services.rest.RestClient; @@ -68,11 +78,15 @@ @Stateless public class SolrIndexService { + public static final int EVENTLOG_ENTRY_FLUSH_COUNT = 16; public static final int DEFAULT_MAX_SEARCH_RESULT = 9999; // limiting the // total // number of hits public static final int DEFAULT_PAGE_SIZE = 100; // default docs in one page + @PersistenceContext(unitName = "org.imixs.workflow.jpa") + private EntityManager manager; + @Inject @ConfigProperty(name = "solr.server", defaultValue = "http://solr:8983") private String host; @@ -95,10 +109,12 @@ public class SolrIndexService { @Inject private SchemaService schemaService; - + @Inject - private AdminPService adminPService; + private EventLogService eventLogService; + @Inject + private AdminPService adminPService; private RestClient restClient; @@ -118,8 +134,10 @@ public void init() { } /** - * The Init method verifies the status of the solr core. If no core is found, - * than the method creates a new empty core + * This method verifies the schema of the Solr core. If field definitions have + * change, a schema update is posted to the Solr rest API. + *

+ * The method assumes that a core is already created with a manageable schema. * * @param setupEvent * @throws RestAPIException @@ -131,84 +149,152 @@ public void setup(@Observes SetupEvent setupEvent) throws RestAPIException { // try to get the schma of the core... try { String existingSchema = restClient.get(host + "/api/cores/" + core + "/schema"); - logger.info("...core = OK "); + logger.info("...core - OK "); // update schema updateSchema(existingSchema); } catch (RestAPIException e) { // no schema found - logger.severe("...no solr core '" + core + "' found!"); + logger.severe("...no solr core '" + core + "' found - verify the solr instance!"); throw e; } } /** - * Creates a new solr core and updates the schema defintion + * Updates the schema definition of an existing Solr core. + *

+ * The schema definition is build by the method builUpdateSchema(). The + * updateSchema adds or replaces field definitions depending on the fieldList + * definitions provided by the Imixs SchemaService. See the method + * builUpdateSchema() for details. *

- * In case no core yet exits, the method tries to create a new one. This - * typically is necessary after first deployment. + * The method asumes that a core already exits. Otherwise an exception is + * thrown. * - * @param prop - * @return - * @throws IOException - * @throws Exception + * @param schema + * - existing schema defintion + * @return - an update Schema definition to be POST to the Solr rest api. + * @throws RestAPIException */ public void updateSchema(String schema) throws RestAPIException { + + // create the schema.... - String schemaUpdate = createUpdateSchemaJSONRequest(schema); + String schemaUpdate = buildUpdateSchema(schema); // test if the schemaUdpate contains instructions.... if (!"{}".equals(schemaUpdate)) { String uri = host + "/api/cores/" + core + "/schema"; - logger.info("...update schema '" + core + "':"); - logger.info("..." + schemaUpdate); + logger.info("...updating schema '" + core + "':"); + logger.finest("..." + schemaUpdate); restClient.post(uri, schemaUpdate, "application/json"); - + logger.info("...schema update - successfull "); // force rebuild index rebuildIndex(); } else { - logger.info("...schema = OK "); + logger.info("...schema - OK "); } } /** - * This method adds a collection of documents to the Lucene solr index. The + * This method adds a collection of documents to the Lucene Solr index. The * documents are added immediately to the index. Calling this method within a * running transaction leads to a uncommitted reads in the index. For * transaction control, it is recommended to use instead the the method - * updateDocumetns() which takes care of uncommitted reads. + * solrUpdateService.updateDocuments() which takes care of uncommitted reads. *

* This method is used by the JobHandlerRebuildIndex only. * * @param documents * of ItemCollections to be indexed * @throws RestAPIException - * @throws IndexException */ - public void updateDocumentsUncommitted(List documents) throws RestAPIException { + public void indexDocuments(List documents) throws RestAPIException { long ltime = System.currentTimeMillis(); - + if (documents == null || documents.size() == 0) { // no op! return; } else { - - String xmlRequest = createAddDocumentsXMLRequest(documents); + + String xmlRequest = buildAddDoc(documents); String uri = host + "/solr/" + core + "/update"; logger.info("...update documents '" + core + "':"); restClient.post(uri, xmlRequest, "text/xml"); } - + + if (logger.isLoggable(Level.FINE)) { + logger.fine("... update index block in " + (System.currentTimeMillis() - ltime) + " ms (" + documents.size() + + " workitems total)"); + } + } + + /** + * This method adds a single document to the Lucene Solr index. The document is + * added immediately to the index. Calling this method within a running + * transaction leads to a uncommitted reads in the index. For transaction + * control, it is recommended to use instead the the method + * solrUpdateService.updateDocuments() which takes care of uncommitted reads. + * + * @param documents + * of ItemCollections to be indexed + * @throws RestAPIException + */ + public void indexDocument(ItemCollection document) throws RestAPIException { + List col = new ArrayList(); + col.add(document); + indexDocuments(col); + } + + /** + * This method removes a collection of documents from the Lucene Solr index. + * + * @param documents + * of collection of UniqueIDs to be removed from the index + * @throws RestAPIException + */ + public void removeDocuments(List documents) throws RestAPIException { + long ltime = System.currentTimeMillis(); + + if (documents == null || documents.size() == 0) { + // no op! + return; + } else { + StringBuffer xmlDelete = new StringBuffer(); + xmlDelete.append(""); + for (String id : documents) { + xmlDelete.append("" + id + ""); + } + xmlDelete.append(""); + String xmlRequest = xmlDelete.toString(); + String uri = host + "/solr/" + core + "/update"; + logger.info("...delete documents '" + core + "':"); + restClient.post(uri, xmlRequest, "text/xml"); + } + if (logger.isLoggable(Level.FINE)) { logger.fine("... update index block in " + (System.currentTimeMillis() - ltime) + " ms (" + documents.size() + " workitems total)"); } } - /** - * This method forces an update of the full text index. + * This method removes a single document from the Lucene Solr index. + * + * @param document + * - UniqueID of the document to be removed from the index + * + * @throws RestAPIException + */ + public void removeDocument(String id) throws RestAPIException { + List col = new ArrayList(); + col.add(id); + removeDocuments(col); + } + + /** + * This method forces an update of the full text index. */ public void rebuildIndex() { // now starting index job.... @@ -218,18 +304,73 @@ public void rebuildIndex() { job.replaceItemValue("job", AdminPService.JOB_REBUILD_INDEX); adminPService.createJob(job); } - - + /** - * This method returns a JSON structure to update an existing Solr schema. The - * method adds all fields into a solr update definition that did not yet exist - * in the current schema. + * This method post a search query and returns the result. + * + * @param searchterm + * @return + * @throws QueryException + */ + public String query(String searchTerm) throws QueryException { + + logger.info("...search solr index: " + searchTerm + "..."); + + // URL Encode the query string.... + try { + String uri = host + "/solr/" + core + "/query?q=" + URLEncoder.encode(searchTerm, "UTF-8"); + + logger.info("... uri=" + uri); + String result = restClient.get(uri); + + return result; + } catch (RestAPIException | UnsupportedEncodingException e) { + + logger.severe("Solr search error: " + e.getMessage()); + throw new QueryException(QueryException.QUERY_NOT_UNDERSTANDABLE, e.getMessage(), e); + } + + } + + /** + * This method builds a JSON structure to be used to update an existing Solr + * schema. The method adds or replaces field definitions into a solr update + * schema. + *

+ * The param oldSchema contains the current schema definition of the core. + *

+ * In Solr there a two field types defining if the value of a field is stored + * and returned by a + *

+ * {"add-field":{name=field1, type=text_general, stored=true, docValues=true}} + *

+ * For both cases the values are stored in the lucene index and returned by a + * query. + *

+ *

+ * Stored fields (stored=true) are row orientated. That means that like in a sql + * table the values are stored based on the ID of the document. *

- * The param schema contains the current schema definition of the core. + * In difference the docValues are stored column orientated (forward index). The + * values are ordered based on the search term. For features like sorting, + * grouping or faceting, docValues increase the performance in general. So it + * may look like docValues are the better choice. But one important different is + * how the values are stored. In case of a stored field with multi-values, the + * values are exactly stored in the same order as they were indexed. DocValues + * instead are sorted and reordered. So this will falsify the result of a + * document returned by a query. + *

+ * In Imixs-Workflow we use the stored attribute to return parts of a + * document at query time. We call this a document-stub which contains only a + * subset of fields. Later we load the full document from the SQL database. As + * stored fields in our workflow application are also often used for sorting we + * combine both attributes. In case of a non-stored field we set also + * docValues=false to avoid unnecessary storing of fields. * + * @see https://lucene.apache.org/solr/guide/8_0/docvalues.html * @return */ - protected String createUpdateSchemaJSONRequest(String oldSchema) { + protected String buildUpdateSchema(String oldSchema) { StringBuffer updateSchema = new StringBuffer(); List fieldListStore = schemaService.getFieldListStore(); @@ -239,24 +380,30 @@ protected String createUpdateSchemaJSONRequest(String oldSchema) { // remove white space from oldSchema to simplify string compare... oldSchema = oldSchema.replace(" ", ""); + + logger.finest("......old schema="+oldSchema); + updateSchema.append("{"); // finally add the default content field - addFieldDefinitionToUpdateSchema(updateSchema, oldSchema, "content", "text_general", false); + addFieldDefinitionToUpdateSchema(updateSchema, oldSchema, "content", "text_general", false, false); // add each field from the fieldListAnalyse for (String field : fieldListAnalyse) { boolean store = fieldListStore.contains(field); - addFieldDefinitionToUpdateSchema(updateSchema, oldSchema, field, "text_general", store); + // text_general - docValues are not supported! + addFieldDefinitionToUpdateSchema(updateSchema, oldSchema, field, "text_general", store, false); } // add each field from the fieldListNoAnalyse for (String field : fieldListNoAnalyse) { boolean store = fieldListStore.contains(field); - addFieldDefinitionToUpdateSchema(updateSchema, oldSchema, field, "strings", store); + // strings - docValues are supported so set it independently from the store flag + // to true. This is to increase sort and grouping performance + addFieldDefinitionToUpdateSchema(updateSchema, oldSchema, field, "strings", store, true); } // finally add the $uniqueid field - addFieldDefinitionToUpdateSchema(updateSchema, oldSchema, "$uniqueid", "string", true); + addFieldDefinitionToUpdateSchema(updateSchema, oldSchema, "$uniqueid", "string", true, false); // remove last , int lastComma = updateSchema.lastIndexOf(","); @@ -268,11 +415,11 @@ protected String createUpdateSchemaJSONRequest(String oldSchema) { } /** - * This method returns a XNK structure to add new documents into the solr index. + * This method returns a XML structure to add new documents into the solr index. * * @return xml content to update documents */ - protected String createAddDocumentsXMLRequest(List documents) { + protected String buildAddDoc(List documents) { List fieldList = schemaService.getFieldList(); List fieldListAnalyse = schemaService.getFieldListAnalyse(); @@ -339,39 +486,54 @@ protected String createAddDocumentsXMLRequest(List documents) { } /** - * This method adds a 'add-field' object to an updateSchema. + * This method adds a field definition object to an updateSchema. *

* In case the same field already exists in the oldSchema then the method will - * not add the field to the update schema. + * replace the field. In case id does not exist, the field definition is added + * to the update schema. *

- * Example: {name=$workflowsummary, type=text_general, stored=true} - * + * Example: + *

+ * add-field:{name:"$workflowsummary", type:"text_general", stored:true, docValues:false}
+ * replace-field:{name:"$workflowstatus", type:"strings", stored:true, docValues:true} *

- * NOTE: The test here is very week (simple indexOf) and may cause problems in - * the future. TODO optimize the schema compare method. + * To verify the existence of the field we parse the fieldname in the old schema + * definition. * * @param updateSchema - * - a stringBuffer containing the update schema + * - a stringBuffer to build the update schema * @param oldSchema - * - the current schema definition + * - the existing schema definition * @param name * - field name * @param type * - field type * @param store * - boolean store field - * @param addComma - * - true if a ',' should be added to the end of the updateSchema. + * @param docValue + * - true if docValues should be set to true * */ private void addFieldDefinitionToUpdateSchema(StringBuffer updateSchema, String oldSchema, String name, String type, - boolean store) { + boolean store, boolean docvalue) { - String fieldDefinition = "{\"name\":\"" + name + "\",\"type\":\"" + type + "\",\"stored\":" + store + "}"; - // test if this field discription already exists - if (!oldSchema.contains(fieldDefinition)) { + String fieldDefinition = "{\"name\":\"" + name + "\",\"type\":\"" + type + "\",\"stored\":" + store + + ",\"docValues\":" + docvalue + "}"; + + // test if this field already exists in the old schema. If not we add the new + // field to the schema with the 'add-field' command. + // If it already exits, than we need to replace the definition with + // 'replace-field'. + String testSchemaField = "{\"name\":\"" + name + "\","; + if (oldSchema == null || !oldSchema.contains(testSchemaField)) { // add new field to updateSchema.... updateSchema.append("\"add-field\":" + fieldDefinition + ","); + } else { + // the field exists in the schema - so replace it if the definition has changed + if (!oldSchema.contains(fieldDefinition)) { + updateSchema.append("\"replace-field\":" + fieldDefinition + ","); + } } } @@ -422,4 +584,140 @@ private void addFieldValuesToUpdateRequest(StringBuffer xmlContent, final ItemCo } } + + /** + * This method flushes a given count of eventLogEntries. The method return true + * if no more eventLogEntries exist. + * + * @param count + * the max size of a eventLog engries to remove. + * @return true if the cache was totally flushed. + */ + protected boolean flushEventLogByCount(int count) { + Date lastEventDate = null; + boolean cacheIsEmpty = true; + + long l = System.currentTimeMillis(); + logger.finest("......flush eventlog cache...."); + + List events = eventLogService.findEventsByTopic(count + 1, DocumentService.EVENTLOG_TOPIC_INDEX_ADD, + DocumentService.EVENTLOG_TOPIC_INDEX_REMOVE); + + if (events != null && events.size() > 0) { + try { + + int _counter = 0; + for (EventLog eventLogEntry : events) { + + // lookup the Document Entity... + org.imixs.workflow.engine.jpa.Document doc = manager + .find(org.imixs.workflow.engine.jpa.Document.class, eventLogEntry.getRef()); + + // if the document was found we add/update the index. Otherwise we remove the + // document form the index. + if (doc != null && DocumentService.EVENTLOG_TOPIC_INDEX_ADD.equals(eventLogEntry.getTopic())) { + // add workitem to search index.... + long l2 = System.currentTimeMillis(); + ItemCollection workitem = new ItemCollection(); + workitem.setAllItems(doc.getData()); + if (!workitem.getItemValueBoolean(DocumentService.NOINDEX)) { + indexDocument(workitem); + logger.info("......solr added workitem '" + eventLogEntry.getId() + "' to index in " + + (System.currentTimeMillis() - l2) + "ms"); + } + } else { + long l2 = System.currentTimeMillis(); + removeDocument(eventLogEntry.getId()); + logger.info("......solr remove workitem '" + eventLogEntry.getId() + "' from index in " + + (System.currentTimeMillis() - l2) + "ms"); + } + + // remove the eventLogEntry. + lastEventDate = eventLogEntry.getCreated().getTime(); + eventLogService.removeEvent(eventLogEntry); + + // break? + _counter++; + if (_counter >= count) { + // we skipp the last one if the maximum was reached. + cacheIsEmpty = false; + break; + } + } + + } catch (RestAPIException e) { + logger.warning("...unable to flush lucene event log: " + e.getMessage()); + // We just log a warning here and close the flush mode to no longer block the + // writer. + // NOTE: maybe throwing a IndexException would be an alternative: + // + // throw new IndexException(IndexException.INVALID_INDEX, "Unable to update + // lucene search index", + // luceneEx); + return true; + } + } + + logger.fine("...flushEventLog - " + events.size() + " events in " + (System.currentTimeMillis() - l) + + " ms - last log entry: " + lastEventDate); + + return cacheIsEmpty; + + } + + /** + * Flush the EventLog cache. This method is called by the LuceneSerachService + * only. + *

+ * The method flushes the cache in smaller blocks of the given junkSize. to + * avoid a heap size problem. The default flush size is 16. The eventLog cache + * is tracked by the flag 'dirtyIndex'. + *

+ * issue #439 - The method returns false if the event log contains more entries + * as defined by the given JunkSize. In this case the caller should recall the + * method which runs always in a new transaction. The goal of this mechanism is + * to reduce the event log even in cases the outer transaction breaks. + * + * @see LuceneSearchService + * @return true if the the complete event log was flushed. If false the method + * must be recalled. + */ + @TransactionAttribute(value = TransactionAttributeType.REQUIRES_NEW) + public boolean flushEventLog(int junkSize) { + long total = 0; + long count = 0; + boolean dirtyIndex = true; + long l = System.currentTimeMillis(); + + while (dirtyIndex) { + try { + dirtyIndex = !flushEventLogByCount(EVENTLOG_ENTRY_FLUSH_COUNT); + if (dirtyIndex) { + total = total + EVENTLOG_ENTRY_FLUSH_COUNT; + count = count + EVENTLOG_ENTRY_FLUSH_COUNT; + if (count >= 100) { + logger.finest("...flush event log: " + total + " entries in " + (System.currentTimeMillis() - l) + + "ms..."); + count = 0; + } + + // issue #439 + // In some cases the flush method runs endless. + // experimental code: we break the flush method after 1024 flushs + // maybe we can remove this hard break + if (total >= junkSize) { + logger.finest("...flush event: Issue #439 -> total count >=" + total + + " flushEventLog will be continued..."); + return false; + } + } + + } catch (IndexException e) { + logger.warning("...unable to flush lucene event log: " + e.getMessage()); + return true; + } + } + return true; + } + } diff --git a/imixs-workflow-index-solr/src/main/java/org/imixs/workflow/engine/solr/SolrSearchService.java b/imixs-workflow-index-solr/src/main/java/org/imixs/workflow/engine/solr/SolrSearchService.java index ecbea5288..7fe0d52c4 100644 --- a/imixs-workflow-index-solr/src/main/java/org/imixs/workflow/engine/solr/SolrSearchService.java +++ b/imixs-workflow-index-solr/src/main/java/org/imixs/workflow/engine/solr/SolrSearchService.java @@ -27,21 +27,28 @@ package org.imixs.workflow.engine.solr; +import java.io.StringReader; import java.util.ArrayList; import java.util.List; +import java.util.NoSuchElementException; import java.util.logging.Logger; import javax.annotation.security.DeclareRoles; import javax.annotation.security.RolesAllowed; import javax.ejb.Stateless; import javax.inject.Inject; +import javax.json.Json; +import javax.json.stream.JsonParser; +import javax.json.stream.JsonParser.Event; +import org.eclipse.microprofile.config.inject.ConfigProperty; import org.imixs.workflow.ItemCollection; import org.imixs.workflow.engine.DocumentService; import org.imixs.workflow.engine.index.DefaultOperator; import org.imixs.workflow.engine.index.SchemaService; import org.imixs.workflow.engine.index.SearchService; import org.imixs.workflow.exceptions.QueryException; +import org.imixs.workflow.util.JSONParser; /** * This session ejb provides a service to search the solr index. @@ -64,70 +71,21 @@ public class SolrSearchService implements SearchService { // number of hits public static final int DEFAULT_PAGE_SIZE = 100; // default docs in one page + @Inject + @ConfigProperty(name = "solr.core", defaultValue = "imixs-workflow") + private String core; + @Inject private SchemaService schemaService; + @Inject + private SolrIndexService solarIndexService; + @Inject private DocumentService documentService; private static Logger logger = Logger.getLogger(SolrSearchService.class.getName()); - - - /** - * Returns a collection of documents matching the provided search term. The - * provided search team will we extended with a users roles to test the read - * access level of each workitem matching the search term. - * - * @param sSearchTerm - * @return collection of search result - * @throws QueryException - */ -// @Override -// public List search(String sSearchTerm) throws QueryException { -// // no sort order -// return search(sSearchTerm, DEFAULT_MAX_SEARCH_RESULT, 0, null, DefaultOperator.AND); -// } - - /** - * Returns a collection of documents matching the provided search term. The - * provided search team will we extended with a users roles to test the read - * access level of each workitem matching the search term. - * - * @param pageSize - * - docs per page - * @param pageIndex - * - page number - * - * @return collection of search result - * @throws QueryException - */ -// @Override -// public List search(String sSearchTerm, int pageSize, int pageIndex) throws QueryException { -// // no sort order -// return search(sSearchTerm, pageSize, pageIndex, null, null); -// } - - /** - * Returns a collection of documents matching the provided search term. The term - * will be extended with the current users roles to test the read access level - * of each workitem matching the search term. - *

- * The method returns the full loaded documents. If you only want to search for - * document stubs use instead the method - *

- * search(String searchTerm, int pageSize, int pageIndex, Sort sortOrder, - Operator defaultOperator, boolean loadStubs) - *

- * - */ -// @Override -// public List search(String sSearchTerm, int pageSize, int pageIndex, -// org.imixs.workflow.engine.index.SortOrder sortOrder, DefaultOperator defaultOperator) -// throws QueryException { -// return search(sSearchTerm, pageSize, pageIndex, sortOrder, defaultOperator, false); -// } - /** * Returns a collection of documents matching the provided search term. The term * will be extended with the current users roles to test the read access level @@ -189,12 +147,57 @@ public List search(String searchTerm, int pageSize, int pageInde if (searchTerm == null || "".equals(searchTerm)) { return workitems; } + + // post query.... + String result = solarIndexService.query(searchTerm); + logger.finest("......Result = " + result); + + if (result != null && !result.isEmpty()) { + List documentStubs = parseJSONQueyResult(result); + // now we extract the docs and build ItemCollections from it.... + + } + + logger.fine("...search result computed in " + (System.currentTimeMillis() - ltime) + " ms - loadStubs=" + + loadStubs); + + return workitems; + } + + /** + * This method extracts the docs from a Solr JSON query result + * + * @param json - solr query response (JSON) + * @return List of ItemCollection objects + */ + protected List parseJSONQueyResult(String json) { + List result= new ArrayList(); + String response = JSONParser.getKey("response", json); + String docs = JSONParser.getKey("docs", response); - logger.fine("...search result computed in " + (System.currentTimeMillis() - ltime) + " ms - loadStubs=" - + loadStubs); + // now we go through the json docs structure and build ItemCollections ...... - return workitems; + logger.info("docs=" + docs); + JsonParser parser = Json.createParser(new StringReader(docs)); + + Event event = null; + while (true) { + + try { + event = parser.next(); // START_OBJECT + if (event == null) { + return null; + } + if (event.name().equals(Event.KEY_NAME.toString())) { + String jsonkey = parser.getString(); + break; + } + } catch (NoSuchElementException e) { + return null; + } + } + return result; } /** @@ -223,5 +226,4 @@ public int getTotalHits(final String _searchTerm, final int _maxResult, final De return 0; } - } diff --git a/imixs-workflow-index-solr/src/main/java/org/imixs/workflow/engine/solr/SolrUpdateService.java b/imixs-workflow-index-solr/src/main/java/org/imixs/workflow/engine/solr/SolrUpdateService.java index 16fa9185c..77a0b7745 100644 --- a/imixs-workflow-index-solr/src/main/java/org/imixs/workflow/engine/solr/SolrUpdateService.java +++ b/imixs-workflow-index-solr/src/main/java/org/imixs/workflow/engine/solr/SolrUpdateService.java @@ -30,7 +30,6 @@ import java.util.List; import java.util.logging.Logger; -import javax.annotation.PostConstruct; import javax.ejb.Stateless; import javax.inject.Inject; @@ -62,18 +61,7 @@ public class SolrUpdateService implements UpdateService { private static Logger logger = Logger.getLogger(SolrUpdateService.class.getName()); - /** - * PostContruct event - The method loads the lucene index properties from the - * imixs.properties file from the classpath. If no properties are defined the - * method terminates. - * - */ - @PostConstruct - void init() { - - logger.finest("...... "); - - } + /** * This method adds a collection of documents to the Lucene index. The documents @@ -92,7 +80,7 @@ void init() { @Override public void updateIndex(List documents) { try { - solrIndexService.updateDocumentsUncommitted(documents); + solrIndexService.indexDocuments(documents); } catch (RestAPIException e) { logger.severe("Failed to update document collection: " + e.getMessage()); throw new IndexException(IndexException.INVALID_INDEX, "Unable to update solr search index", e); @@ -101,8 +89,15 @@ public void updateIndex(List documents) { @Override public void updateIndex() { - // TODO - logger.warning(" unimplemented !!!!"); + long ltime = System.currentTimeMillis(); + // flush eventlog (see issue #411) + int flushCount = 0; + while (solrIndexService.flushEventLog(2048) == false) { + // repeat flush.... + flushCount = +2048; + logger.info("...flush event log: " + flushCount + " entries updated in " + + (System.currentTimeMillis() - ltime) + "ms ..."); + } } } diff --git a/imixs-workflow-index-solr/src/test/java/org/imixs/workflow/engine/solr/TestParseSolrJSONResult.java b/imixs-workflow-index-solr/src/test/java/org/imixs/workflow/engine/solr/TestParseSolrJSONResult.java new file mode 100644 index 000000000..f7b253186 --- /dev/null +++ b/imixs-workflow-index-solr/src/test/java/org/imixs/workflow/engine/solr/TestParseSolrJSONResult.java @@ -0,0 +1,73 @@ +package org.imixs.workflow.engine.solr; + +import java.util.List; +import java.util.logging.Logger; + +import org.imixs.workflow.ItemCollection; +import org.imixs.workflow.exceptions.ModelException; +import org.imixs.workflow.exceptions.PluginException; +import org.junit.Before; +import org.junit.Test; + +import junit.framework.Assert; + +/** + * Test the WorkflowService method parseJSONQueyResult from SolrSerachService + * + * @author rsoika + * + */ +public class TestParseSolrJSONResult { + + @SuppressWarnings("unused") + private final static Logger logger = Logger.getLogger(TestParseSolrJSONResult.class.getName()); + + SolrSearchService solrSearchService=null; + + @Before + public void setUp() throws PluginException, ModelException { + + solrSearchService=new SolrSearchService(); + } + + + /** + * Test + * + */ + @Test + public void testParseResult() { + + String testString = "{\n" + + " \"responseHeader\":{\n" + + " \"status\":0,\n" + + " \"QTime\":4,\n" + + " \"params\":{\n" + + " \"q\":\"*:*\",\n" + + " \"_\":\"1567286252995\"}},\n" + + " \"response\":{\"numFound\":2,\"start\":0,\"docs\":[\n" + + " {\n" + + " \"type\":[\"model\"],\n" + + " \"id\":\"3a182d18-33d9-4951-8970-d9eaf9d337ff\",\n" + + " \"_modified\":[20190831211617],\n" + + " \"_created\":[20190831211617],\n" + + " \"_version_\":1643418672068296704},\n" + + " {\n" + + " \"type\":[\"adminp\"],\n" + + " \"id\":\"60825929-4d7d-4346-9333-afd7dbfca457\",\n" + + " \"_modified\":[20190831211618],\n" + + " \"_created\":[20190831211618],\n" + + " \"_version_\":1643418672172105728}]\n" + + " }}"; + + + + +// List result=solrSearchService.parseJSONQueyResult(testString); +// +// +// Assert.assertTrue(result.size()==2); + + } + +}