From 91624b37fd8f8d9678674570640378dcb3cf4e1c Mon Sep 17 00:00:00 2001 From: Ray Lee Date: Thu, 3 Jan 2019 14:35:39 -0800 Subject: [PATCH] DRYD-560: Initial implementation of ES reindexing event listener. --- .../nuxeo/nuxeo-platform-listener/build.xml | 9 +- .../nuxeo/nuxeo-platform-listener/pom.xml | 3 +- .../reindex/build.properties | 1 + .../nuxeo-platform-listener/reindex/build.xml | 104 ++++++++++++++++ .../nuxeo-platform-listener/reindex/pom.xml | 51 ++++++++ .../services/listener/Reindex.java | 115 ++++++++++++++++++ .../services/listener/ReindexSupport.java | 71 +++++++++++ .../src/main/resources/META-INF/MANIFEST.MF | 13 ++ .../OSGI-INF/deployment-fragment.xml | 10 ++ .../main/resources/OSGI-INF/event-contrib.xml | 19 +++ 10 files changed, 392 insertions(+), 4 deletions(-) create mode 100644 3rdparty/nuxeo/nuxeo-platform-listener/reindex/build.properties create mode 100644 3rdparty/nuxeo/nuxeo-platform-listener/reindex/build.xml create mode 100644 3rdparty/nuxeo/nuxeo-platform-listener/reindex/pom.xml create mode 100644 3rdparty/nuxeo/nuxeo-platform-listener/reindex/src/main/java/org/collectionspace/services/listener/Reindex.java create mode 100644 3rdparty/nuxeo/nuxeo-platform-listener/reindex/src/main/java/org/collectionspace/services/listener/ReindexSupport.java create mode 100644 3rdparty/nuxeo/nuxeo-platform-listener/reindex/src/main/resources/META-INF/MANIFEST.MF create mode 100644 3rdparty/nuxeo/nuxeo-platform-listener/reindex/src/main/resources/OSGI-INF/deployment-fragment.xml create mode 100644 3rdparty/nuxeo/nuxeo-platform-listener/reindex/src/main/resources/OSGI-INF/event-contrib.xml diff --git a/3rdparty/nuxeo/nuxeo-platform-listener/build.xml b/3rdparty/nuxeo/nuxeo-platform-listener/build.xml index f5d1eb238..87dd0d928 100644 --- a/3rdparty/nuxeo/nuxeo-platform-listener/build.xml +++ b/3rdparty/nuxeo/nuxeo-platform-listener/build.xml @@ -115,29 +115,32 @@ + - + + - + + - + diff --git a/3rdparty/nuxeo/nuxeo-platform-listener/pom.xml b/3rdparty/nuxeo/nuxeo-platform-listener/pom.xml index 913bdb8c5..093c67f39 100644 --- a/3rdparty/nuxeo/nuxeo-platform-listener/pom.xml +++ b/3rdparty/nuxeo/nuxeo-platform-listener/pom.xml @@ -13,6 +13,7 @@ services.3rdparty.nuxeo.listener + reindex updateobjectlocationonmove updaterelationsondelete updateimagederivatives @@ -34,7 +35,7 @@ org.nuxeo.ecm.platform nuxeo-platform-filemanager-api - ${nuxeo.platform.version} + ${nuxeo.platform.version} diff --git a/3rdparty/nuxeo/nuxeo-platform-listener/reindex/build.properties b/3rdparty/nuxeo/nuxeo-platform-listener/reindex/build.properties new file mode 100644 index 000000000..ee625cf41 --- /dev/null +++ b/3rdparty/nuxeo/nuxeo-platform-listener/reindex/build.properties @@ -0,0 +1 @@ +listener.module.name=reindex diff --git a/3rdparty/nuxeo/nuxeo-platform-listener/reindex/build.xml b/3rdparty/nuxeo/nuxeo-platform-listener/reindex/build.xml new file mode 100644 index 000000000..c8b2ee135 --- /dev/null +++ b/3rdparty/nuxeo/nuxeo-platform-listener/reindex/build.xml @@ -0,0 +1,104 @@ + + + CollectionSpace Nuxeo listener component type + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/3rdparty/nuxeo/nuxeo-platform-listener/reindex/pom.xml b/3rdparty/nuxeo/nuxeo-platform-listener/reindex/pom.xml new file mode 100644 index 000000000..0fd43ef90 --- /dev/null +++ b/3rdparty/nuxeo/nuxeo-platform-listener/reindex/pom.xml @@ -0,0 +1,51 @@ + + + 4.0.0 + + org.collectionspace.services.3rdparty.nuxeo.listener + org.collectionspace.services + 5.2-SNAPSHOT + + + UTF-8 + + org.collectionspace.services.listener.reindex + org.collectionspace.services.listener.reindex + http://maven.apache.org + + + + + org.collectionspace.services + org.collectionspace.services.common + ${project.version} + + + + + + + src/main/resources + true + + + + + org.apache.maven.plugins + maven-jar-plugin + + + src/main/resources/META-INF/MANIFEST.MF + + ${eclipseVersion} + 2 + + + + + + + + diff --git a/3rdparty/nuxeo/nuxeo-platform-listener/reindex/src/main/java/org/collectionspace/services/listener/Reindex.java b/3rdparty/nuxeo/nuxeo-platform-listener/reindex/src/main/java/org/collectionspace/services/listener/Reindex.java new file mode 100644 index 000000000..285ffeac9 --- /dev/null +++ b/3rdparty/nuxeo/nuxeo-platform-listener/reindex/src/main/java/org/collectionspace/services/listener/Reindex.java @@ -0,0 +1,115 @@ +package org.collectionspace.services.listener; + +import java.util.Iterator; +import java.util.List; + +import org.apache.commons.collections.ListUtils; +import org.apache.commons.lang3.StringUtils; +import org.nuxeo.ecm.core.api.DocumentModel; +import org.nuxeo.ecm.core.api.LifeCycleConstants; +import org.nuxeo.ecm.core.api.event.DocumentEventTypes; +import org.nuxeo.ecm.core.event.Event; +import org.nuxeo.ecm.core.event.EventBundle; +import org.nuxeo.ecm.core.event.PostCommitEventListener; +import org.nuxeo.ecm.core.event.impl.DocumentEventContext; +import org.nuxeo.elasticsearch.ElasticSearchComponent; +import org.nuxeo.elasticsearch.api.ElasticSearchService; +import org.nuxeo.runtime.api.Framework; + +/** + * Event listener that triggers reindexing of records in Elasticsearch when an associated record + * is created/updated/deleted. When a record is created or updated Nuxeo will automatically + * reindex it in ElasticSearch, but Nuxeo does not know about other records that may also need to + * be reindexed; for example, if a related record denormalizes data from the updated record at + * index time. + */ +public class Reindex implements PostCommitEventListener { + // FIXME: This listener runs asynchronously post-commit, so that reindexing records after a + // save does not hold up the save. In order to make it async, this class does not extend + // AbstractCSEventListenerImpl, because AbstractCSEventListenerImpl does not implement + // PostCommitEventListener (DRYD-477). Therefore this listener is not able to use the + // isRegistered method of AbstractCSEventListenerImpl to determine if it has been registered to + // run for the current tenant. Instead, it relies on the ReindexSupport listener, which does + // extend AbstractCSEventListenerImpl, to set a property in the event context that is used to + // determine if this listener should run. This means that this listener will be considered to + // be registered if and only if the ReindexSupport listener is registered. + + public static final String IS_REGISTERED_KEY = "Reindex.IS_REGISTERED"; + public static final String PREV_COVERAGE_KEY = "Reindex.PREV_COVERAGE"; + public static final String PREV_PUBLISH_TO_KEY = "Reindex.PREV_PUBLISH_TO"; + + @Override + public void handleEvent(EventBundle events) { + // When a media record is created, reindex the material item that is referenced by its + // coverage field. + + // When a media record is updated and the coverage changed, reindex both the old and new + // referenced material items. + + // When a media record is deleted, reindex the material item that was referenced by its + // coverage field. + + // TODO: Make this configurable. This is currently hardcoded to the needs of the material + // profile/Material Order application. + + if (Framework.isBooleanPropertyTrue("elasticsearch.enabled") && events.size() > 0) { + Iterator iter = events.iterator(); + + while (iter.hasNext()) { + Event event = iter.next(); + DocumentEventContext eventContext = (DocumentEventContext) event.getContext(); + Boolean isRegistered = (Boolean) eventContext.getProperty(IS_REGISTERED_KEY); + + if (isRegistered != null && isRegistered == true) { + DocumentModel doc = eventContext.getSourceDocument(); + String docType = doc.getType(); + String eventName = event.getName(); + + if (docType.startsWith("Media")) { + if ( + eventName.equals(DocumentEventTypes.DOCUMENT_CREATED) || + eventName.equals(DocumentEventTypes.DOCUMENT_UPDATED) + ) { + String prevCoverage = (String) eventContext.getProperty(PREV_COVERAGE_KEY); + String coverage = (String) doc.getProperty("media_common", "coverage"); + + List prevPublishTo = (List) eventContext.getProperty(PREV_PUBLISH_TO_KEY); + List publishTo = (List) doc.getProperty("media_materials", "publishToList"); + + if (doc.getCurrentLifeCycleState().equals(LifeCycleConstants.DELETED_STATE)) { + reindex(doc.getRepositoryName(), coverage); + } + else if ( + !ListUtils.isEqualList(prevPublishTo, publishTo) || + !StringUtils.equals(prevCoverage, coverage) + ) { + if (!StringUtils.equals(prevCoverage, coverage)) { + reindex(doc.getRepositoryName(), prevCoverage); + } + + reindex(doc.getRepositoryName(), coverage); + } + } + else if (eventName.equals(DocumentEventTypes.DOCUMENT_REMOVED)) { + String prevCoverage = (String) eventContext.getProperty(PREV_COVERAGE_KEY); + + reindex(doc.getRepositoryName(), prevCoverage); + } + } + } + } + } + } + + private void reindex(String repositoryName, String refName) { + if (StringUtils.isEmpty(refName)) { + return; + } + + String escapedRefName = refName.replace("'", "\\'"); + String query = String.format("SELECT ecm:uuid FROM Materialitem WHERE collectionspace_core:refName = '%s'", escapedRefName); + + ElasticSearchComponent es = (ElasticSearchComponent) Framework.getService(ElasticSearchService.class); + es.runReindexingWorker(repositoryName, query); + } +} diff --git a/3rdparty/nuxeo/nuxeo-platform-listener/reindex/src/main/java/org/collectionspace/services/listener/ReindexSupport.java b/3rdparty/nuxeo/nuxeo-platform-listener/reindex/src/main/java/org/collectionspace/services/listener/ReindexSupport.java new file mode 100644 index 000000000..313e227d4 --- /dev/null +++ b/3rdparty/nuxeo/nuxeo-platform-listener/reindex/src/main/java/org/collectionspace/services/listener/ReindexSupport.java @@ -0,0 +1,71 @@ +package org.collectionspace.services.listener; + +import java.io.Serializable; +import java.util.List; + +import org.collectionspace.services.nuxeo.listener.AbstractCSEventListenerImpl; +import org.nuxeo.ecm.core.api.DocumentModel; +import org.nuxeo.ecm.core.api.event.CoreEventConstants; +import org.nuxeo.ecm.core.api.event.DocumentEventTypes; +import org.nuxeo.ecm.core.event.Event; +import org.nuxeo.ecm.core.event.impl.DocumentEventContext; +import org.nuxeo.runtime.api.Framework; + +/** + * Event listener that stores the values of fields of interest before documents are updated or + * deleted. This is necessary because the previous/deleted doument model will not be available + * to a post-modification/deletion event listener. Storing the previous/deleted values allows + * the post-modification/deletion event listener to take action if a field value was changed, + * or if a document was deleted that had a certain field value. + * + * This is a separate class from the Reindex listener, because the Reindex listener should be + * async and post-commit, so it must implement PostCommitEventListener. This listener must be + * synchronous and pre-commit, so it must implement EventListener. Nuxeo does not support + * a single class that implements both PostCommitEventListener and EventListener (such a listener + * will only run synchronously). + */ +public class ReindexSupport extends AbstractCSEventListenerImpl { + + @Override + public void handleEvent(Event event) { + // When a media record is about to be updated, store the value of the coverage and + // publishToList fields. + + // When a media record is about to be removed, store the value of the coverage field. + + // TODO: Make this configurable. This is currently hardcoded to the needs of the material + // profile/Material Order application. + + if (isRegistered(event)) { + DocumentEventContext eventContext = (DocumentEventContext) event.getContext(); + + // Set a property if this listener is registered for the current tenant. This allows + // the Reindex listener to determine if it should run (since it is async, it cannot + // extend AbstractCSEventListenerImpl, so it cannot use the isRegistered method). + + eventContext.setProperty(Reindex.IS_REGISTERED_KEY, true); + + if (Framework.isBooleanPropertyTrue("elasticsearch.enabled")) { + DocumentModel doc = eventContext.getSourceDocument(); + String docType = doc.getType(); + String eventName = event.getName(); + + if (docType.startsWith("Media")) { + if (eventName.equals(DocumentEventTypes.BEFORE_DOC_UPDATE)) { + DocumentModel previousDoc = (DocumentModel) eventContext.getProperty(CoreEventConstants.PREVIOUS_DOCUMENT_MODEL); + String coverage = (String) previousDoc.getProperty("media_common", "coverage"); + List publishTo = (List) previousDoc.getProperty("media_materials", "publishToList"); + + eventContext.setProperty(Reindex.PREV_COVERAGE_KEY, coverage); + eventContext.setProperty(Reindex.PREV_PUBLISH_TO_KEY, (Serializable) publishTo); + } + else if (eventName.equals(DocumentEventTypes.ABOUT_TO_REMOVE)) { + String coverage = (String) doc.getProperty("media_common", "coverage"); + + eventContext.setProperty(Reindex.PREV_COVERAGE_KEY, coverage); + } + } + } + } + } +} diff --git a/3rdparty/nuxeo/nuxeo-platform-listener/reindex/src/main/resources/META-INF/MANIFEST.MF b/3rdparty/nuxeo/nuxeo-platform-listener/reindex/src/main/resources/META-INF/MANIFEST.MF new file mode 100644 index 000000000..932d325a3 --- /dev/null +++ b/3rdparty/nuxeo/nuxeo-platform-listener/reindex/src/main/resources/META-INF/MANIFEST.MF @@ -0,0 +1,13 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 1 +Bundle-Name: org.collectionspace.services.listener.reindex +Bundle-SymbolicName: org.collectionspace.services.listener.reindex;singleton:=true +Bundle-Version: 1.0.0 +Bundle-Localization: plugin +Bundle-Vendor: Nuxeo +Require-Bundle: org.nuxeo.runtime, + org.nuxeo.ecm.core.api, + org.nuxeo.ecm.core, + org.nuxeo.ecm.webapp.core +Provide-Package: org.collectionspace.services.listener.reindex +Nuxeo-Component: OSGI-INF/event-contrib.xml diff --git a/3rdparty/nuxeo/nuxeo-platform-listener/reindex/src/main/resources/OSGI-INF/deployment-fragment.xml b/3rdparty/nuxeo/nuxeo-platform-listener/reindex/src/main/resources/OSGI-INF/deployment-fragment.xml new file mode 100644 index 000000000..270abbd5f --- /dev/null +++ b/3rdparty/nuxeo/nuxeo-platform-listener/reindex/src/main/resources/OSGI-INF/deployment-fragment.xml @@ -0,0 +1,10 @@ + + + + + + ${bundle.fileName} + + + + diff --git a/3rdparty/nuxeo/nuxeo-platform-listener/reindex/src/main/resources/OSGI-INF/event-contrib.xml b/3rdparty/nuxeo/nuxeo-platform-listener/reindex/src/main/resources/OSGI-INF/event-contrib.xml new file mode 100644 index 000000000..3be10bf5a --- /dev/null +++ b/3rdparty/nuxeo/nuxeo-platform-listener/reindex/src/main/resources/OSGI-INF/event-contrib.xml @@ -0,0 +1,19 @@ + + + + + documentCreated + documentModified + documentRemoved + + + + + aboutToCreate + aboutToRemove + beforeDocumentModification + + + -- 2.47.3