]> git.aero2k.de Git - tmp/jakarta-migration.git/commitdiff
DRYD-559: Initial implementation of ES document writer.
authorRay Lee <rhlee@berkeley.edu>
Thu, 3 Jan 2019 07:53:12 +0000 (23:53 -0800)
committerRay Lee <rhlee@berkeley.edu>
Thu, 3 Jan 2019 07:53:12 +0000 (23:53 -0800)
3rdparty/nuxeo/build.xml
3rdparty/nuxeo/nuxeo-platform-elasticsearch/build.xml [new file with mode: 0644]
3rdparty/nuxeo/nuxeo-platform-elasticsearch/pom.xml [new file with mode: 0644]
3rdparty/nuxeo/nuxeo-platform-elasticsearch/src/main/java/org/collectionspace/services/nuxeo/elasticsearch/CSJsonESDocumentWriter.java [new file with mode: 0644]
3rdparty/nuxeo/nuxeo-platform-elasticsearch/src/main/resources/META-INF/MANIFEST.MF [new file with mode: 0644]
3rdparty/nuxeo/nuxeo-platform-elasticsearch/src/main/resources/OSGI-INF/deployment-fragment.xml [new file with mode: 0644]
3rdparty/nuxeo/nuxeo-platform-elasticsearch/src/main/resources/OSGI-INF/elasticsearch-contrib.xml [new file with mode: 0644]
3rdparty/nuxeo/pom.xml

index a1190749a4db0adb46fbca9b1b83ba752bbd684e..b65277bb4d9b930d0c10582be823ce99c2cd197d 100644 (file)
         </copy>
         <copy todir="${jee.server.cspace}/cspace/config/services" overwrite="true">
             <fileset file="${basedir}/nuxeo-server/${nuxeo.release}/config/proto-elasticsearch-extension.xml"/>
-        </copy>        
-               
+        </copy>
+
         <ant antfile="nuxeo-platform-collectionspace/build.xml" target="deploy" inheritall="false"/>
         <ant antfile="nuxeo-platform-listener/build.xml" target="deploy" inheritall="false"/>
-        <!-- Disabled due to integration issues between ImageMagick and Ghostscript on Linux systems        
+        <ant antfile="nuxeo-platform-elasticsearch/build.xml" target="deploy" inheritall="false"/>
+        <!-- Disabled due to integration issues between ImageMagick and Ghostscript on Linux systems
         <ant antfile="nuxeo-platform-thumbnail/build.xml" target="deploy" inheritall="false"/>
         -->
     </target>
         <delete file="${jee.server.cspace}/cspace/config/services/proto-datasource-config.xml" failonerror="false"/>
         <ant antfile="nuxeo-platform-collectionspace/build.xml" target="undeploy" inheritall="false"/>
         <ant antfile="nuxeo-platform-listener/build.xml" target="undeploy" inheritall="false"/>
+        <ant antfile="nuxeo-platform-elasticsearch/build.xml" target="undeploy" inheritall="false"/>
         <ant antfile="nuxeo-platform-quote/build.xml" target="undeploy" inheritall="false"/>
         <ant antfile="nuxeo-platform-quote-api/build.xml" target="undeploy" inheritall="false"/>
         <!-- Disabled due to integration issues between ImageMagick and Ghostscript on Linux systems
diff --git a/3rdparty/nuxeo/nuxeo-platform-elasticsearch/build.xml b/3rdparty/nuxeo/nuxeo-platform-elasticsearch/build.xml
new file mode 100644 (file)
index 0000000..cab6c95
--- /dev/null
@@ -0,0 +1,100 @@
+<project name="org.collectionspace.services.3rdparty.nuxeo.elasticsearch" default="package" basedir=".">
+  <description>
+    CollectionSpace Nuxeo Elasticsearch component
+  </description>
+  <!-- set global properties for this build -->
+  <property name="services.trunk" value="../../.."/>
+  <!-- environment should be declared before reading build.properties -->
+  <property environment="env" />
+  <property file="${services.trunk}/build.properties" />
+  <property name="mvn.opts" value="-V" />
+  <property name="src" location="src"/>
+  <property name="build" location="build"/>
+  <property name="dist"  location="dist"/>
+
+  <property name="nuxeo.collectionspace.elasticsearch.jar"
+    value="org.collectionspace.services.3rdparty.nuxeo.elasticsearch-${cspace.release}.jar"/>
+
+  <property name="nuxeo.collectionspace.elasticsearch.jar.all"
+    value="org.collectionspace.services.3rdparty.nuxeo.elasticsearch-*.jar"/>
+
+  <condition property="osfamily-unix">
+    <os family="unix" />
+  </condition>
+  <condition property="osfamily-windows">
+    <os family="windows" />
+  </condition>
+
+  <target name="init" >
+    <!-- Create the time stamp -->
+    <tstamp/>
+    <!-- Create the build directory structure used by compile -->
+    <mkdir dir="${build}"/>
+  </target>
+
+  <target name="package" depends="package-unix,package-windows"
+    description="Package CollectionSpace Services" />
+  <target name="package-unix" if="osfamily-unix">
+    <exec executable="mvn" failonerror="true">
+      <arg value="package" />
+      <arg value="-Dmaven.test.skip=true" />
+      <arg value="-f" />
+      <arg value="${basedir}/pom.xml" />
+      <arg value="-N" />
+      <arg value="${mvn.opts}" />
+    </exec>
+  </target>
+  <target name="package-windows" if="osfamily-windows">
+    <exec executable="cmd" failonerror="true">
+      <arg value="/c" />
+      <arg value="mvn" />
+      <arg value="package" />
+      <arg value="-Dmaven.test.skip=true" />
+      <arg value="-f" />
+      <arg value="${basedir}/pom.xml" />
+      <arg value="-N" />
+      <arg value="${mvn.opts}" />
+    </exec>
+  </target>
+
+  <target name="install" depends="install-unix,install-windows"
+    description="Install" />
+  <target name="install-unix" if="osfamily-unix">
+    <exec executable="mvn" failonerror="true">
+      <arg value="install" />
+      <arg value="-Dmaven.test.skip=true" />
+      <arg value="-f" />
+      <arg value="${basedir}/pom.xml" />
+      <arg value="-N" />
+      <arg value="${mvn.opts}" />
+    </exec>
+  </target>
+  <target name="install-windows" if="osfamily-windows">
+    <exec executable="cmd" failonerror="true">
+      <arg value="/c" />
+      <arg value="mvn" />
+      <arg value="install" />
+      <arg value="-Dmaven.test.skip=true" />
+      <arg value="-f" />
+      <arg value="${basedir}/pom.xml" />
+      <arg value="-N" />
+      <arg value="${mvn.opts}" />
+    </exec>
+  </target>
+
+  <target name="deploy" depends="install"
+    description="deploy collectionspace Elasticsearch component in ${jee.server.nuxeo}">
+    <copy file="${basedir}/target/${nuxeo.collectionspace.elasticsearch.jar}"
+      todir="${jee.deploy.nuxeo.plugins}"/>
+  </target>
+
+  <target name="undeploy"
+    description="undeploy collectionspace Elasticsearch component from ${jee.server.nuxeo}">
+    <delete>
+      <fileset dir="${jee.deploy.nuxeo.plugins}">
+        <include name="${nuxeo.collectionspace.elasticsearch.jar.all}"/>
+      </fileset>
+    </delete>
+  </target>
+
+</project>
diff --git a/3rdparty/nuxeo/nuxeo-platform-elasticsearch/pom.xml b/3rdparty/nuxeo/nuxeo-platform-elasticsearch/pom.xml
new file mode 100644 (file)
index 0000000..517a6d5
--- /dev/null
@@ -0,0 +1,60 @@
+<?xml version="1.0"?>
+<project
+       xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
+       xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+       <modelVersion>4.0.0</modelVersion>
+       <parent>
+               <artifactId>org.collectionspace.services.3rdparty.nuxeo</artifactId>
+               <groupId>org.collectionspace.services</groupId>
+               <version>5.2-SNAPSHOT</version>
+       </parent>
+       <artifactId>org.collectionspace.services.3rdparty.nuxeo.elasticsearch</artifactId>
+       <name>org.collectionspace.services.3rdparty.nuxeo.elasticsearch</name>
+       <url>http://maven.apache.org</url>
+       <properties>
+               <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+       </properties>
+
+       <dependencies>
+        <dependency>
+            <groupId>org.nuxeo.ecm.core</groupId>
+            <artifactId>nuxeo-core-api</artifactId>
+            <version>${nuxeo.core.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.nuxeo.ecm.automation</groupId>
+            <artifactId>nuxeo-automation-io</artifactId>
+            <version>${nuxeo.core.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.collectionspace.services</groupId>
+            <artifactId>org.collectionspace.services.common</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+       </dependencies>
+
+    <build>
+               <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+                <filtering>true</filtering>
+            </resource>
+               </resources>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <configuration>
+                    <archive>
+                        <manifestFile> src/main/resources/META-INF/MANIFEST.MF </manifestFile>
+                        <manifestEntries>
+                            <Bundle-Version>${eclipseVersion}</Bundle-Version>
+                            <Bundle-ManifestVersion>2</Bundle-ManifestVersion>
+                        </manifestEntries>
+                    </archive>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>
diff --git a/3rdparty/nuxeo/nuxeo-platform-elasticsearch/src/main/java/org/collectionspace/services/nuxeo/elasticsearch/CSJsonESDocumentWriter.java b/3rdparty/nuxeo/nuxeo-platform-elasticsearch/src/main/java/org/collectionspace/services/nuxeo/elasticsearch/CSJsonESDocumentWriter.java
new file mode 100644 (file)
index 0000000..810dd27
--- /dev/null
@@ -0,0 +1,269 @@
+package org.collectionspace.services.nuxeo.elasticsearch;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import javax.ws.rs.core.HttpHeaders;
+
+import org.apache.commons.lang3.StringUtils;
+import org.codehaus.jackson.JsonGenerator;
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.node.ObjectNode;
+import org.codehaus.jackson.node.TextNode;
+
+import org.collectionspace.services.common.api.RefNameUtils;
+
+import org.nuxeo.ecm.automation.jaxrs.io.documents.JsonESDocumentWriter;
+import org.nuxeo.ecm.core.api.CoreSession;
+import org.nuxeo.ecm.core.api.DocumentModel;
+import org.nuxeo.ecm.core.api.DocumentModelList;
+
+public class CSJsonESDocumentWriter extends JsonESDocumentWriter {
+    private static ObjectMapper objectMapper = new ObjectMapper();
+
+    @Override
+    public void writeDoc(JsonGenerator jg, DocumentModel doc, String[] schemas,
+            Map<String, String> contextParameters, HttpHeaders headers)
+            throws IOException {
+
+        // Compute and store fields that should be indexed with this document in ElasticSearch.
+        // TODO: Make this configurable. This is currently hardcoded for the materials profile and
+        // the Material Order application.
+
+        ObjectNode denormValues = objectMapper.createObjectNode();
+
+        String docType = doc.getType();
+
+        if (docType.startsWith("Materialitem")) {
+            CoreSession session = doc.getCoreSession();
+
+            // Store the csids of media records that reference this material authority item via the
+            // coverage field.
+
+            String refName = (String) doc.getProperty("collectionspace_core", "refName");
+
+            if (StringUtils.isNotEmpty(refName)) {
+                String escapedRefName = refName.replace("'", "\\'");
+                String mediaQuery = String.format("SELECT * FROM Media WHERE media_common:coverage = '%s' AND ecm:currentLifeCycleState = 'project' AND collectionspace_core:tenantId = '2000' ORDER BY media_common:identificationNumber", escapedRefName);
+
+                DocumentModelList mediaDocs = session.query(mediaQuery);
+                List<JsonNode> mediaCsids = new ArrayList<JsonNode>();
+
+                if (mediaDocs.size() > 0) {
+                    Iterator<DocumentModel> iterator = mediaDocs.iterator();
+
+                    while (iterator.hasNext()) {
+                        DocumentModel mediaDoc = iterator.next();
+
+                        if (isMediaPublished(mediaDoc)) {
+                            String mediaCsid = (String) mediaDoc.getName();
+                        
+                            mediaCsids.add(new TextNode(mediaCsid));
+                        }
+                    }
+                }
+
+                denormValues.putArray("mediaCsid").addAll(mediaCsids);
+            }
+
+            // Compute the title of the record for the public browser, and store it so that it can
+            // be used for sorting ES query results.
+
+            String title = computeTitle(doc);
+
+            if (title != null) {
+                denormValues.put("title", title);
+            }
+            
+            List<Map<String, Object>> termGroups = (List<Map<String, Object>>) doc.getProperty("materials_common", "materialTermGroupList");
+            List<String> commercialNames = findTermDisplayNamesWithFlag(termGroups, "commercial");
+            List<String> commonNames = findTermDisplayNamesWithFlag(termGroups, "common");
+
+            // Find and store the commercial names and common names for this item. This simplifies
+            // search and display in the Material Order application.
+
+            if (commercialNames.size() > 0) {
+                denormValues.putArray("commercialNames").addAll(jsonNodes(commercialNames));
+            }
+
+            if (commonNames.size() > 0) {
+                denormValues.putArray("commonNames").addAll(jsonNodes(commonNames));
+            }
+        }
+
+        // Below is sample code for denormalizing fields from the computed current location (place
+        // item) into collection object documents. This was written for the public browser
+        // prototype for public art.
+
+        /*
+        if (docType.startsWith("CollectionObject")) {
+            CoreSession session = doc.getCoreSession();
+
+            String refName = (String) doc.getProperty("collectionobjects_common", "computedCurrentLocation");
+
+            if (StringUtils.isNotEmpty(refName)) {
+                String escapedRefName = refName.replace("'", "\\'");
+                String placeQuery = String.format("SELECT * FROM PlaceitemTenant5000 WHERE places_common:refName = '%s'", escapedRefName);
+
+                DocumentModelList placeDocs = session.query(placeQuery, 1);
+
+                if (placeDocs.size() > 0) {
+                    DocumentModel placeDoc = placeDocs.get(0);
+
+                    String placementType = (String) placeDoc.getProperty("places_publicart:placementType").getValue();
+
+                    if (placementType != null) {
+                        denormValues.put("placementType", placementType);
+                    }
+
+                    Property geoRefGroup;
+
+                    try {
+                        geoRefGroup = placeDoc.getProperty("places_common:placeGeoRefGroupList/0");
+                    } catch (PropertyNotFoundException e) {
+                        geoRefGroup = null;
+                    }
+
+                    if (geoRefGroup != null) {
+                        Double decimalLatitude = (Double) geoRefGroup.getValue("decimalLatitude");
+                        Double decimalLongitude = (Double) geoRefGroup.getValue("decimalLongitude");
+
+                        if (decimalLatitude != null && decimalLongitude != null) {
+                            ObjectNode geoPointNode = objectMapper.createObjectNode();
+
+                            geoPointNode.put("lat", decimalLatitude);
+                            geoPointNode.put("lon", decimalLongitude);
+
+                            denormValues.put("geoPoint", geoPointNode);
+                        }
+                    }
+                }
+            }
+
+            String uri = (String) doc.getProperty("collectionobjects_core", "uri");
+            String csid = uri.substring(uri.lastIndexOf('/') + 1);
+            String mediaQuery = String.format("SELECT media_common:blobCsid, media_common:title FROM Relation WHERE relations_common:subjectCsid = '%s' AND relations_common:objectDocumentType = 'Media'", csid);
+
+            DocumentModelList mediaDocs = session.query(mediaQuery, 1);
+
+            if (mediaDocs.size() > 0) {
+
+            }
+        }
+        */
+
+        jg.writeStartObject();
+
+        writeSystemProperties(jg, doc);
+        writeSchemas(jg, doc, schemas);
+        writeContextParameters(jg, doc, contextParameters);
+        writeDenormValues(jg, doc, denormValues);
+
+        jg.writeEndObject();
+        jg.flush();
+    }
+
+    public void writeDenormValues(JsonGenerator jg, DocumentModel doc, ObjectNode denormValues) throws IOException {
+        if (denormValues != null && denormValues.size() > 0) {
+            if (jg.getCodec() == null) {
+                jg.setCodec(objectMapper);
+            }
+
+            Iterator<Map.Entry<String, JsonNode>> entries = denormValues.getFields();
+
+            while (entries.hasNext()) {
+                Map.Entry<String, JsonNode> entry = entries.next();
+
+                jg.writeFieldName("collectionspace_denorm:" + entry.getKey());
+                jg.writeTree(entry.getValue());
+            }
+        }
+    }
+
+    /**
+     * Compute a title for the public browser. This needs to be indexed in ES so that it can
+     * be used for sorting. (Even if it's just extracting the primary value.)
+     */
+    private String computeTitle(DocumentModel doc) {
+        List<Map<String, Object>> termGroups = (List<Map<String, Object>>) doc.getProperty("materials_common", "materialTermGroupList");
+        String primaryDisplayName = null;
+
+        if (termGroups.size() > 0) {
+            Map<String, Object> primaryTermGroup = termGroups.get(0);
+            primaryDisplayName = (String) primaryTermGroup.get("termDisplayName");
+        }
+
+        return primaryDisplayName;
+    }
+
+    private String findFirstTermDisplayNameWithFlag(List<Map<String, Object>> termGroups, String flagShortId) {
+        String termDisplayName = null;
+
+        for (Map<String, Object> termGroup : termGroups) {
+            String termFlag = (String) termGroup.get("termFlag");
+
+            if (termFlag != null && termFlag.contains("(" + flagShortId + ")")) {
+                String candidateTermDisplayName = (String) termGroup.get("termDisplayName");
+
+                if (StringUtils.isNotEmpty(candidateTermDisplayName)) {
+                    termDisplayName = candidateTermDisplayName;
+                    break;
+                }
+            }
+        }
+
+        return termDisplayName;
+    }
+
+    private List<String> findTermDisplayNamesWithFlag(List<Map<String, Object>> termGroups, String flagShortId) {
+        List<String> termDisplayNames = new ArrayList<String>();
+
+        for (Map<String, Object> termGroup : termGroups) {
+            String termFlag = (String) termGroup.get("termFlag");
+
+            if (termFlag != null && termFlag.contains("(" + flagShortId + ")")) {
+                String candidateTermDisplayName = (String) termGroup.get("termDisplayName");
+
+                if (StringUtils.isNotEmpty(candidateTermDisplayName)) {
+                    termDisplayNames.add(candidateTermDisplayName);
+                }
+            }
+        }
+
+        return termDisplayNames;
+    }
+
+    private boolean isMediaPublished(DocumentModel mediaDoc) {
+        List<String> publishToValues = (List<String>) mediaDoc.getProperty("media_materials", "publishToList");
+        boolean isPublished = false;
+        
+        for (int i=0; i<publishToValues.size(); i++) {
+            String value = publishToValues.get(i);
+            String shortId = RefNameUtils.getItemShortId(value);
+
+            if (shortId.equals("all") || shortId.equals("materialorder")) {
+                isPublished = true;
+                break;
+            }
+        }
+
+        return isPublished;
+    }
+
+    private List<JsonNode> jsonNodes(List<String> values) {
+        List<JsonNode> nodes = new ArrayList<JsonNode>();
+        Iterator<String> iterator = values.iterator();
+
+        while (iterator.hasNext()) {
+            String value = iterator.next();
+
+            nodes.add(new TextNode(value));
+        }
+
+        return nodes;
+    }
+}
diff --git a/3rdparty/nuxeo/nuxeo-platform-elasticsearch/src/main/resources/META-INF/MANIFEST.MF b/3rdparty/nuxeo/nuxeo-platform-elasticsearch/src/main/resources/META-INF/MANIFEST.MF
new file mode 100644 (file)
index 0000000..cf42ab7
--- /dev/null
@@ -0,0 +1,13 @@
+Manifest-Version: 1.0
+Bundle-ManifestVersion: 1
+Bundle-Name: org.collectionspace.nuxeo.elasticsearch
+Bundle-SymbolicName: org.collectionspace.nuxeo.elasticsearch;singleton:=true
+Bundle-Version: 1.0.0
+Bundle-Localization: plugin
+Bundle-Vendor: Nuxeo
+Require-Bundle: org.nuxeo.runtime,
+ org.nuxeo.ecm.core.api,
+ org.nuxeo.ecm.core,
+ org.nuxeo.ecm.webapp.core
+Provide-Package: org.collectionspace.nuxeo.elasticsearch
+Nuxeo-Component: OSGI-INF/elasticsearch-contrib.xml,
diff --git a/3rdparty/nuxeo/nuxeo-platform-elasticsearch/src/main/resources/OSGI-INF/deployment-fragment.xml b/3rdparty/nuxeo/nuxeo-platform-elasticsearch/src/main/resources/OSGI-INF/deployment-fragment.xml
new file mode 100644 (file)
index 0000000..270abbd
--- /dev/null
@@ -0,0 +1,10 @@
+<?xml version="1.0"?>
+<fragment>
+
+  <extension target="application#MODULE">
+    <module>
+      <java>${bundle.fileName}</java>
+    </module>
+  </extension>
+  
+</fragment>
diff --git a/3rdparty/nuxeo/nuxeo-platform-elasticsearch/src/main/resources/OSGI-INF/elasticsearch-contrib.xml b/3rdparty/nuxeo/nuxeo-platform-elasticsearch/src/main/resources/OSGI-INF/elasticsearch-contrib.xml
new file mode 100644 (file)
index 0000000..962ddec
--- /dev/null
@@ -0,0 +1,7 @@
+<?xml version="1.0"?>
+<component name="org.collectionspace.nuxeo.elasticsearch">
+  <extension target="org.nuxeo.elasticsearch.ElasticSearchComponent"
+    point="elasticSearchDocWriter">
+    <writer class="org.collectionspace.services.nuxeo.elasticsearch.CSJsonESDocumentWriter" />
+  </extension>
+</component>
index 3dd1652beecc1ead6de309f261f57e7f9a968503..44b1f3e9dfd2a5abafdcddfe1c27a7f1234d71da 100644 (file)
@@ -16,7 +16,8 @@
     <modules>
                <module>nuxeo-platform-collectionspace</module>
                <module>nuxeo-platform-listener</module>
-               <!-- disabled in v4.2 build do to test failures during upgrade to Nuxeo 6 
+               <module>nuxeo-platform-elasticsearch</module>
+               <!-- disabled in v4.2 build do to test failures during upgrade to Nuxeo 6
                <module>nuxeo-platform-quote-api</module>
                <module>nuxeo-platform-quote</module>
                -->