From: Ray Lee Date: Fri, 5 Jul 2019 00:27:38 +0000 (-0700) Subject: DRYD-656: Implement ES index rebuild at /index/elasticsearch. X-Git-Url: https://git.aero2k.de/?a=commitdiff_plain;h=9a14e4cfe0e46d5a5fdd48e237774c36a05381c2;p=tmp%2Fjakarta-migration.git DRYD-656: Implement ES index rebuild at /index/elasticsearch. --- diff --git a/services/JaxRsServiceProvider/src/main/java/org/collectionspace/services/jaxrs/CSpaceResteasyBootstrap.java b/services/JaxRsServiceProvider/src/main/java/org/collectionspace/services/jaxrs/CSpaceResteasyBootstrap.java index c60dc923a..82244d93e 100644 --- a/services/JaxRsServiceProvider/src/main/java/org/collectionspace/services/jaxrs/CSpaceResteasyBootstrap.java +++ b/services/JaxRsServiceProvider/src/main/java/org/collectionspace/services/jaxrs/CSpaceResteasyBootstrap.java @@ -116,12 +116,11 @@ public class CSpaceResteasyBootstrap extends ResteasyBootstrap { if (isElasticsearchIndexed && servicesRepoDomainName != null && servicesRepoDomainName.trim().isEmpty() == false) { String repositoryName = ConfigUtils.getRepositoryName(tenantBinding, servicesRepoDomainName); - String docType = serviceBinding.getObject().getName(); - String tenantQualifiedDocType = NuxeoUtils.getTenantQualifiedDocType(tenantBinding.getId(), docType); + String docType = NuxeoUtils.getTenantQualifiedDocType(tenantBinding.getId(), serviceBinding.getObject().getName()); - logger.log(Level.INFO, String.format("%tc [INFO] Starting Elasticsearch reindexing for docType %s in repository %s", new Date(), tenantQualifiedDocType, repositoryName)); + logger.log(Level.INFO, String.format("%tc [INFO] Starting Elasticsearch reindexing for docType %s in repository %s", new Date(), docType, repositoryName)); - es.runReindexingWorker(repositoryName, String.format("SELECT ecm:uuid FROM %s", tenantQualifiedDocType)); + es.runReindexingWorker(repositoryName, String.format("SELECT ecm:uuid FROM %s", docType)); } } } diff --git a/services/common/src/main/cspace/config/services/tenants/tenant-bindings-proto-unified.xml b/services/common/src/main/cspace/config/services/tenants/tenant-bindings-proto-unified.xml index cd664d914..c3543c8e0 100644 --- a/services/common/src/main/cspace/config/services/tenants/tenant-bindings-proto-unified.xml +++ b/services/common/src/main/cspace/config/services/tenants/tenant-bindings-proto-unified.xml @@ -25,25 +25,25 @@ - ptStartingWildcard true - maxListItemsReturnedLimitOnJdbcQueries 40 - jdbcQueriesAreTenantIdRestricted @@ -141,10 +141,10 @@ - + - default-domain @@ -153,7 +153,7 @@ - + default-domain - + default-domain - + @@ -374,7 +374,7 @@ - + @@ -385,13 +385,14 @@ - fulltext - SELECT ecm:uuid, ecm:primaryType FROM Document WHERE ecm:isProxy = 0 ORDER BY ecm:uuid - - - elasticsearch - SELECT ecm:uuid, ecm:primaryType FROM Document WHERE ecm:isProxy = 0 ORDER BY ecm:uuid - + fulltext + SELECT ecm:uuid, ecm:primaryType FROM Document WHERE ecm:isProxy = 0 ORDER BY ecm:uuid + + + elasticsearch + + SELECT ecm:uuid FROM %s + @@ -411,7 +412,7 @@ - + @@ -530,17 +531,17 @@ - - org.collectionspace.services.relation.nuxeo.RelationValidatorHandler @@ -639,8 +640,8 @@ - default-domain org.collectionspace.services.contact.nuxeo.ContactDocumentModelHandler @@ -676,9 +677,9 @@ authRef addressGroupList/*/addressStateOrProvince - - @@ -784,11 +785,11 @@ - org.collectionspace.services.authorization.storage.PermissionRoleDocumentHandler - @@ -817,7 +818,7 @@ org.collectionspace.services.account.storage.AccountRoleDocumentHandler - @@ -845,11 +846,11 @@ - org.collectionspace.services.authorization.storage.PermissionRoleDocumentHandler - @@ -877,11 +878,11 @@ - org.collectionspace.services.account.storage.AccountRoleDocumentHandler - diff --git a/services/common/src/main/java/org/collectionspace/services/common/document/AbstractDocumentHandlerImpl.java b/services/common/src/main/java/org/collectionspace/services/common/document/AbstractDocumentHandlerImpl.java index 8ffa1d259..aa296e796 100644 --- a/services/common/src/main/java/org/collectionspace/services/common/document/AbstractDocumentHandlerImpl.java +++ b/services/common/src/main/java/org/collectionspace/services/common/document/AbstractDocumentHandlerImpl.java @@ -40,10 +40,10 @@ import org.slf4j.LoggerFactory; * * $LastChangedRevision: $ * $LastChangedDate: $ - * @param - * @param - * @param - * @param + * @param + * @param + * @param + * @param */ @SuppressWarnings({"unchecked", "rawtypes"}) public abstract class AbstractDocumentHandlerImpl @@ -51,13 +51,13 @@ public abstract class AbstractDocumentHandlerImpl /** The logger. */ private final Logger logger = LoggerFactory.getLogger(AbstractDocumentHandlerImpl.class); - + /** The properties. */ private Map properties = new HashMap(); - + /** The doc filter. */ private DocumentFilter docFilter = null; - + /** The service context. */ private ServiceContext serviceContext; @@ -69,12 +69,12 @@ public abstract class AbstractDocumentHandlerImpl } abstract protected String getRefnameDisplayName(DocumentWrapper docWrapper); - + /* * Should return a reference name for the wrapper object */ abstract protected RefName.RefNameInterface getRefName(DocumentWrapper docWrapper, String tenantName, String serviceName); - + /* (non-Javadoc) * @see org.collectionspace.services.common.document.DocumentHandler#getServiceContext() */ @@ -161,16 +161,16 @@ public abstract class AbstractDocumentHandlerImpl validate(action); prepareDelete(); break; - + case SYNC: prepareSync(); break; - + case WORKFLOW: logger.error("Should never get to this code path. If you did, there is a bug in the code."); Thread.dumpStack(); break; - + default: logger.error("Should never get to this code path. If you did, there is a bug in the code."); Thread.dumpStack(); @@ -227,7 +227,7 @@ public abstract class AbstractDocumentHandlerImpl @Override final public boolean handle(Action action, DocumentWrapper wrapDoc) throws Exception { boolean result = true; - + switch (action) { case CREATE: handleCreate((DocumentWrapper) wrapDoc); @@ -248,25 +248,25 @@ public abstract class AbstractDocumentHandlerImpl case DELETE: result = handleDelete((DocumentWrapper) wrapDoc); break; - + case SYNC: result = handleSync((DocumentWrapper) wrapDoc); - break; - + break; + case WORKFLOW: logger.error("Should never get to this code path. If you did, there is a bug in the code."); Thread.dumpStack(); break; - + default: logger.error("Should never get to this code path. If you did, there is a bug in the code."); Thread.dumpStack(); break; } - + return result; } - + @Override public void sanitize(DocumentWrapper wrapDoc) { // @@ -306,7 +306,7 @@ public abstract class AbstractDocumentHandlerImpl public boolean handleDelete(DocumentWrapper wrapDoc) throws Exception { return true; } - + /* (non-Javadoc) * @see org.collectionspace.services.common.document.DocumentHandler#handleDelete(org.collectionspace.services.common.document.DocumentWrapper) */ @@ -315,7 +315,7 @@ public abstract class AbstractDocumentHandlerImpl // Do nothing. Subclasses can override if they want/need to. return true; } - + /* (non-Javadoc) * @see org.collectionspace.services.common.document.DocumentHandler#complete(org.collectionspace.services.common.document.DocumentHandler.Action, org.collectionspace.services.common.document.DocumentWrapper) @@ -342,20 +342,20 @@ public abstract class AbstractDocumentHandlerImpl case DELETE: completeDelete((DocumentWrapper) wrapDoc); break; - + case SYNC: completeSync((DocumentWrapper) wrapDoc); break; - + case WORKFLOW: logger.error("Should never get to this code path. If you did, there is a bug in the code."); Thread.dumpStack(); break; - + default: logger.error("Should never get to this code path. If you did, there is a bug in the code."); Thread.dumpStack(); - break; + break; } } @@ -394,13 +394,13 @@ public abstract class AbstractDocumentHandlerImpl @Override public void completeDelete(DocumentWrapper wrapDoc) throws Exception { } - + /* (non-Javadoc) * @see org.collectionspace.services.common.document.DocumentHandler#completeDelete(org.collectionspace.services.common.document.DocumentWrapper) */ @Override public void completeSync(DocumentWrapper wrapDoc) throws Exception { - } + } /* (non-Javadoc) * @see org.collectionspace.services.common.document.DocumentHandler#extractCommonPart(org.collectionspace.services.common.document.DocumentWrapper) @@ -461,7 +461,7 @@ public abstract class AbstractDocumentHandlerImpl @Override public abstract String getQProperty(String prop) throws DocumentException; - /* + /* * Strip Nuxeo's schema name from the start of the field / element name. * (non-Javadoc) * @see org.collectionspace.services.common.document.DocumentHandler#getUnQProperty(java.lang.String) @@ -478,17 +478,22 @@ public abstract class AbstractDocumentHandlerImpl tkz.nextToken(); //skip return tkz.nextToken(); } - + /** * Should return - * @throws Exception - * @throws DocumentException + * @throws Exception + * @throws DocumentException */ @Override public String getDocumentsToIndexQuery(String indexId, String csid) throws DocumentException, Exception { return null; } + @Override + public String getDocumentsToIndexQuery(String indexId, String documentType, String csid) throws DocumentException, Exception { + return null; + } + /* (non-Javadoc) * @see org.collectionspace.services.common.document.DocumentHandler#getServiceContextPath() */ @@ -509,11 +514,11 @@ public abstract class AbstractDocumentHandlerImpl handler.validate(action, serviceContext); } } - + /** * Creates the CMIS query from the service context. Each document handler is responsible for returning a valid CMIS query using the * information in the current service context -which includes things like the query parameters, etc. - * @throws DocumentException + * @throws DocumentException */ @Override public String getCMISQuery(QueryContext queryContext) throws DocumentException { @@ -522,20 +527,20 @@ public abstract class AbstractDocumentHandlerImpl // return null; } - + @Override public boolean isCMISQuery() { return false; } - + @Override public boolean isJDBCQuery() { return false; } - + @Override public Map getJDBCQueryParams() { return new HashMap<>(); } - + } diff --git a/services/common/src/main/java/org/collectionspace/services/common/document/DocumentHandler.java b/services/common/src/main/java/org/collectionspace/services/common/document/DocumentHandler.java index 731dff906..970d4a2a7 100644 --- a/services/common/src/main/java/org/collectionspace/services/common/document/DocumentHandler.java +++ b/services/common/src/main/java/org/collectionspace/services/common/document/DocumentHandler.java @@ -50,9 +50,9 @@ public interface DocumentHandler { public enum Action { CREATE, GET, GET_ALL, UPDATE, DELETE, WORKFLOW, SYNC } - + public Lifecycle getLifecycle(); - + public Lifecycle getLifecycle(String serviceObjectName); /** @@ -75,7 +75,7 @@ public interface DocumentHandler { /** * prepare is called by the client for preparation of stuff before - * invoking repository operation. this is mainly useful for create and + * invoking repository operation. this is mainly useful for create and * update kind of actions * @param action * @throws Exception @@ -86,7 +86,7 @@ public interface DocumentHandler { * updateWorkflowTransition - prepare for a workflow transition */ public void handleWorkflowTransition(ServiceContext ctx, DocumentWrapper wrapDoc, TransitionDef transitionDef) throws Exception; - + /** * prepareCreate processes documents before creating document in repository @@ -122,7 +122,7 @@ public interface DocumentHandler { /** * prepare is called by the client to hand over the document processing task - * @param action + * @param action * @param doc wrapped doc * @throws Exception */ @@ -236,7 +236,7 @@ public interface DocumentHandler { * @throws Exception */ public TL extractCommonPartList(DocumentWrapper docWrap) throws Exception; - + /** * Extract paging info. @@ -246,7 +246,7 @@ public interface DocumentHandler { * @return the tL * @throws Exception the exception */ - public TL extractPagingInfo(TL theCommonList, DocumentWrapper wrapDoc) throws Exception; + public TL extractPagingInfo(TL theCommonList, DocumentWrapper wrapDoc) throws Exception; /** * fillCommonPartList sets list common part of CS object into given document @@ -275,7 +275,7 @@ public interface DocumentHandler { * createDocumentFilter is a factory method to create a document * filter that is relevant to be used with this document handler * and corresponding storage client - * + * * @return */ public DocumentFilter createDocumentFilter(); @@ -323,7 +323,7 @@ public interface DocumentHandler { * getQProperty get qualified property (useful for mapping to repository document property) * @param prop * @return - * @throws DocumentException + * @throws DocumentException */ public String getQProperty(String prop) throws DocumentException; @@ -333,21 +333,28 @@ public interface DocumentHandler { * @return unqualified property */ public String getUnQProperty(String qProp); - + /** * get a query string that will be used to return a set of documents that should be indexed/re-index - * @throws Exception - * @throws DocumentException + * @throws Exception + * @throws DocumentException */ public String getDocumentsToIndexQuery(String indexId, String csid) throws DocumentException, Exception; - + + /** + * get a query string that will be used to return a set of documents that should be indexed/re-index + * @throws Exception + * @throws DocumentException + */ + public String getDocumentsToIndexQuery(String indexId, String documentType, String csid) throws DocumentException, Exception; + /** * Creates the CMIS query from the service context. Each document handler is responsible for returning a valid CMIS query using the * information in the current service context -which includes things like the query parameters, etc. - * @throws DocumentException + * @throws DocumentException */ public String getCMISQuery(QueryContext queryContext) throws DocumentException; - + /** * Returns TRUE if a CMIS query should be used (instead of an NXQL query) */ @@ -357,29 +364,29 @@ public interface DocumentHandler { * Returns TRUE if a JDBC/SQL query should be used (instead of an NXQL query) */ public boolean isJDBCQuery(); - + /** * Returns parameter values, relevant to this document handler, that can be used in JDBC/SQL queries - * + * * @return a set of zero or more parameter values relevant to this handler */ public Map getJDBCQueryParams(); /** - * + * * @throws Exception */ void prepareSync() throws Exception; /** - * + * * @param wrapDoc * @throws Exception */ boolean handleSync(DocumentWrapper wrapDoc) throws Exception; /** - * + * * @param wrapDoc * @throws Exception */ diff --git a/services/common/src/main/java/org/collectionspace/services/nuxeo/client/java/NuxeoRepositoryClientImpl.java b/services/common/src/main/java/org/collectionspace/services/nuxeo/client/java/NuxeoRepositoryClientImpl.java index 6c15294e6..461ae716e 100644 --- a/services/common/src/main/java/org/collectionspace/services/nuxeo/client/java/NuxeoRepositoryClientImpl.java +++ b/services/common/src/main/java/org/collectionspace/services/nuxeo/client/java/NuxeoRepositoryClientImpl.java @@ -64,6 +64,7 @@ import org.collectionspace.services.common.config.TenantBindingConfigReaderImpl; import org.collectionspace.services.common.config.TenantBindingUtils; import org.collectionspace.services.common.storage.PreparedStatementBuilder; import org.collectionspace.services.common.vocabulary.RefNameServiceUtils.AuthorityItemSpecifier; +import org.collectionspace.services.config.service.ServiceBindingType; import org.collectionspace.services.config.tenant.TenantBindingType; import org.collectionspace.services.config.tenant.RepositoryDomainType; @@ -91,6 +92,7 @@ import org.nuxeo.ecm.core.opencmis.impl.server.NuxeoCmisService; import org.nuxeo.elasticsearch.api.ElasticSearchAdmin; import org.nuxeo.elasticsearch.api.ElasticSearchIndexing; import org.nuxeo.elasticsearch.api.ElasticSearchService; +import org.nuxeo.elasticsearch.ElasticSearchComponent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -239,7 +241,7 @@ public class NuxeoRepositoryClientImpl implements RepositoryClient ctx = handler.getServiceContext(); try { - ElasticSearchIndexing esi = Framework.getService(ElasticSearchIndexing.class); - ElasticSearchAdmin esa = Framework.getService(ElasticSearchAdmin.class); - - String queryString = handler.getDocumentsToIndexQuery(indexid, csid); - esa.initIndexes(true); - esa.refresh(); repoSession = getRepositorySession(ctx); - esi.runReindexingWorker(repoSession.getRepositoryName(), queryString); + + ElasticSearchComponent es = (ElasticSearchComponent) Framework.getService(ElasticSearchService.class); + String repositoryName = repoSession.getRepositoryName(); + + logger.info(String.format("Rebuilding Elasticsearch index for repository %s", repositoryName)); + + es.dropAndInitRepositoryIndex(repositoryName); + + TenantBindingConfigReaderImpl tReader = ServiceMain.getInstance().getTenantBindingConfigReader(); + TenantBindingType tenantBinding = tReader.getTenantBinding(ctx.getTenantId()); + + for (ServiceBindingType serviceBinding : tenantBinding.getServiceBindings()) { + Boolean isElasticsearchIndexed = serviceBinding.isElasticsearchIndexed(); + String servicesRepoDomainName = serviceBinding.getRepositoryDomain(); + + if (isElasticsearchIndexed && servicesRepoDomainName != null && servicesRepoDomainName.trim().isEmpty() == false) { + String docType = NuxeoUtils.getTenantQualifiedDocType(tenantBinding.getId(), serviceBinding.getObject().getName()); + String queryString = handler.getDocumentsToIndexQuery(indexid, docType, csid); + + logger.info(String.format("Starting Elasticsearch reindexing for docType %s in repository %s", docType, repositoryName)); + logger.debug(queryString); + + es.runReindexingWorker(repositoryName, queryString); + } + } + result = true; } catch (Throwable e) { rollbackTransaction(repoSession); @@ -331,7 +352,7 @@ public class NuxeoRepositoryClientImpl implements RepositoryClient { private final Logger logger = LoggerFactory.getLogger(IndexDocumentModelHandler.class); - @Override - public String getDocumentsToIndexQuery(String indexId, String csid) throws DocumentException, Exception { + public String getDocumentsToIndexQuery(String indexId, String csid) throws DocumentException, Exception { + return getDocumentsToIndexQuery(indexId, "Document", csid); + } + + @Override + public String getDocumentsToIndexQuery(String indexId, String documentType, String csid) throws DocumentException, Exception { String result = null; - + switch (indexId) { case IndexClient.FULLTEXT_ID: - result = getReindexQuery(indexId, csid); + result = getReindexQuery(indexId, documentType, csid); break; case IndexClient.ELASTICSEARCH_ID: - result = getReindexQuery(indexId, csid); - + result = getReindexQuery(indexId, documentType, csid); + break; } - + if (Tools.isEmpty(result) == true) { String msg = String.format("There is no reindex query in the Index service bindings for index '%s', so we'll use this default query: '%s'", indexId, IndexClient.DEFAULT_REINDEX_QUERY); @@ -74,41 +78,42 @@ public class IndexDocumentModelHandler return result; } - + /** * Reads the Index service bindings to get the query that will be used to find all documents needing * reindexing. - * + * * @param indexId + * @param documentType * @param csid * @return * @throws DocumentException * @throws Exception - * + * * TODO: Use the incoming CSID param to qualify the returned query. */ - private String getReindexQuery(String indexId, String csid) throws DocumentException, Exception { + private String getReindexQuery(String indexId, String documentType, String csid) throws DocumentException, Exception { String result = null; - - // - // Read in the NXQL query to use when performing a full - // - TenantBindingConfigReaderImpl tReader = - ServiceMain.getInstance().getTenantBindingConfigReader(); - ServiceContext ctx = this.getServiceContext(); - - ServiceBindingType reportServiceBinding = tReader.getServiceBinding(ctx.getTenantId(), ctx.getServiceName()); - List queryTypeList = ServiceBindingUtils.getPropertyValueList(reportServiceBinding, indexId); - - if (queryTypeList != null && queryTypeList.isEmpty() == false) { - PropertyItemType propertyItemType = queryTypeList.get(0); - if (propertyItemType != null) { - result = propertyItemType.getValue(); - } - } - - return result; - } -} + // + // Read in the NXQL query to use when performing a full + // + TenantBindingConfigReaderImpl tReader = ServiceMain.getInstance().getTenantBindingConfigReader(); + ServiceContext ctx = this.getServiceContext(); + + ServiceBindingType serviceBinding = tReader.getServiceBinding(ctx.getTenantId(), ctx.getServiceName()); + List queryTypeList = ServiceBindingUtils.getPropertyValueList(serviceBinding, indexId); + + if (queryTypeList != null && queryTypeList.isEmpty() == false) { + PropertyItemType propertyItemType = queryTypeList.get(0); + if (propertyItemType != null) { + String query = propertyItemType.getValue(); + result = String.format(query, documentType); + } + } + + return result; + } + +}