2 * This document is a part of the source code and related artifacts for
3 * CollectionSpace, an open source collections management system for museums and
4 * related institutions:
6 * http://www.collectionspace.org http://wiki.collectionspace.org
8 * Copyright 2009 University of California at Berkeley
10 * Licensed under the Educational Community License (ECL), Version 2.0. You may
11 * not use this file except in compliance with this License.
13 * You may obtain a copy of the ECL 2.0 License at
15 * https://source.collectionspace.org/collection-space/LICENSE.txt
17 * Unless required by applicable law or agreed to in writing, software
18 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
19 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
20 * License for the specific language governing permissions and limitations under
23 package org.collectionspace.services.common.vocabulary;
25 import java.util.ArrayList;
26 import java.util.HashMap;
27 import java.util.Iterator;
28 import java.util.List;
30 import java.util.UUID;
32 import javax.ws.rs.core.Response;
34 import org.nuxeo.ecm.core.api.ClientException;
35 import org.nuxeo.ecm.core.api.DocumentModel;
36 import org.nuxeo.ecm.core.api.DocumentModelList;
37 import org.nuxeo.ecm.core.api.model.Property;
38 import org.nuxeo.ecm.core.api.model.PropertyException;
39 import org.nuxeo.ecm.core.api.model.PropertyNotFoundException;
40 import org.nuxeo.ecm.core.api.model.impl.primitives.StringProperty;
41 import org.slf4j.Logger;
42 import org.slf4j.LoggerFactory;
43 import org.collectionspace.services.client.CollectionSpaceClient;
44 import org.collectionspace.services.client.IQueryManager;
45 import org.collectionspace.services.client.IRelationsManager;
46 import org.collectionspace.services.client.PoxPayloadIn;
47 import org.collectionspace.services.client.PoxPayloadOut;
48 import org.collectionspace.services.client.Profiler;
49 import org.collectionspace.services.common.CSWebApplicationException;
50 import org.collectionspace.services.common.ServiceMain;
51 import org.collectionspace.services.common.ServletTools;
52 import org.collectionspace.services.common.StoredValuesUriTemplate;
53 import org.collectionspace.services.common.UriTemplateFactory;
54 import org.collectionspace.services.common.UriTemplateRegistry;
55 import org.collectionspace.services.common.UriTemplateRegistryKey;
56 import org.collectionspace.services.common.context.ServiceContext;
57 import org.collectionspace.services.common.context.AbstractServiceContextImpl;
58 import org.collectionspace.services.common.api.RefNameUtils;
59 import org.collectionspace.services.common.api.Tools;
60 import org.collectionspace.services.common.api.RefNameUtils.AuthorityTermInfo;
61 import org.collectionspace.services.common.authorityref.AuthorityRefDocList;
62 import org.collectionspace.services.common.config.TenantBindingConfigReaderImpl;
63 import org.collectionspace.services.common.context.ServiceBindingUtils;
64 import org.collectionspace.services.common.document.DocumentException;
65 import org.collectionspace.services.common.document.DocumentFilter;
66 import org.collectionspace.services.common.document.DocumentNotFoundException;
67 import org.collectionspace.services.common.document.DocumentUtils;
68 import org.collectionspace.services.common.document.DocumentWrapper;
69 import org.collectionspace.services.common.query.QueryManager;
70 import org.collectionspace.services.common.relation.RelationUtils;
71 import org.collectionspace.services.common.repository.RepositoryClient;
72 import org.collectionspace.services.nuxeo.client.java.CoreSessionInterface;
73 import org.collectionspace.services.nuxeo.client.java.NuxeoDocumentModelHandler;
74 import org.collectionspace.services.nuxeo.client.java.NuxeoRepositoryClientImpl;
75 import org.collectionspace.services.common.security.SecurityUtils;
76 import org.collectionspace.services.config.service.ServiceBindingType;
77 import org.collectionspace.services.jaxb.AbstractCommonList;
78 import org.collectionspace.services.nuxeo.util.NuxeoUtils;
81 * RefNameServiceUtils is a collection of services utilities related to refName
84 * $LastChangedRevision: $ $LastChangedDate: $
86 public class RefNameServiceUtils {
88 public static enum SpecifierForm {
89 CSID, URN_NAME // Either a CSID or a short ID
92 public static class Specifier {
94 // URN statics for things like urn:cspace:name(grover)
96 final static String URN_PREFIX = "urn:cspace:";
97 final static int URN_PREFIX_LEN = URN_PREFIX.length();
98 final static String URN_PREFIX_NAME = "name(";
99 final static int URN_NAME_PREFIX_LEN = URN_PREFIX_LEN + URN_PREFIX_NAME.length();
100 final static String URN_PREFIX_ID = "id(";
101 final static int URN_ID_PREFIX_LEN = URN_PREFIX_LEN + URN_PREFIX_ID.length();
103 public SpecifierForm form;
106 public Specifier(SpecifierForm form, String value) {
112 * identifier can be a CSID form like a8ad38ec-1d7d-4bf2-bd31 or a URN form like urn:cspace:name(shortid) or urn:cspace:id(a8ad38ec-1d7d-4bf2-bd31)
115 public static Specifier getSpecifier(String identifier) throws CSWebApplicationException {
116 return getSpecifier(identifier, "NO-OP", "NO-OP");
120 * identifier can be a CSID form like a8ad38ec-1d7d-4bf2-bd31 or a URN form like urn:cspace:name(shortid) or urn:cspace:id(a8ad38ec-1d7d-4bf2-bd31)
123 public static Specifier getSpecifier(String identifier, String method, String op) throws CSWebApplicationException {
124 Specifier result = null;
126 if (identifier != null) {
127 if (!identifier.startsWith(URN_PREFIX)) {
128 // We'll assume it is a CSID and complain if it does not match
129 result = new Specifier(SpecifierForm.CSID, identifier);
131 if (identifier.startsWith(URN_PREFIX_NAME, URN_PREFIX_LEN)) {
132 int closeParen = identifier.indexOf(')', URN_NAME_PREFIX_LEN);
133 if (closeParen >= 0) {
134 result = new Specifier(SpecifierForm.URN_NAME,
135 identifier.substring(URN_NAME_PREFIX_LEN, closeParen));
137 } else if (identifier.startsWith(URN_PREFIX_ID, URN_PREFIX_LEN)) {
138 int closeParen = identifier.indexOf(')', URN_ID_PREFIX_LEN);
139 if (closeParen >= 0) {
140 result = new Specifier(SpecifierForm.CSID,
141 identifier.substring(URN_ID_PREFIX_LEN, closeParen));
144 logger.error(method + ": bad or missing specifier!");
145 Response response = Response.status(Response.Status.BAD_REQUEST).entity(
146 op + " failed on bad or missing Authority specifier").type(
147 "text/plain").build();
148 throw new CSWebApplicationException(response);
157 * Creates a refName in the name / shortIdentifier form.
159 * @param shortId a shortIdentifier for an authority or one of its terms
160 * @return a refName for that authority or term, in the name / shortIdentifier form.
161 * If the provided shortIdentifier is null or empty, returns
164 public static String createShortIdURNValue(String shortId) {
165 String result = null;
167 if (shortId != null || !shortId.trim().isEmpty()) {
168 result = String.format("urn:cspace:name(%s)", shortId);
175 * Returns a URN string identifier -e.g., urn:cspace:name(patrick) or urn:cspace:id(579d18a6-b464-4b11-ba3a)
180 public String getURNValue() throws Exception {
181 String result = null;
183 if (form == SpecifierForm.CSID) {
184 result = String.format("urn:cspace:id(%s)", value);
185 } else if (form == SpecifierForm.URN_NAME) {
186 result = String.format("urn:cspace:name(%s)", value);
188 throw new Exception(String.format("Unknown specifier form '%s'.", form));
195 public static class AuthorityItemSpecifier {
196 private Specifier parentSpecifier;
197 private Specifier itemSpecifier;
199 public AuthorityItemSpecifier(Specifier parentSpecifier, Specifier itemSpecifier) {
200 this.parentSpecifier = parentSpecifier;
201 this.itemSpecifier = itemSpecifier;
204 public AuthorityItemSpecifier(SpecifierForm form, String parentCsidOrShortId, String itemCsidOrShortId) {
205 this.parentSpecifier = new Specifier(form, parentCsidOrShortId);
206 this.itemSpecifier = new Specifier(form, itemCsidOrShortId);
209 public Specifier getParentSpecifier() {
210 return this.parentSpecifier;
213 public Specifier getItemSpecifier() {
214 return this.itemSpecifier;
218 public String toString() {
219 String result = "%s/items/%s";
222 result = String.format(result, this.parentSpecifier.getURNValue(), this.itemSpecifier.getURNValue());
223 } catch (Exception e) {
224 result = "Unknown error trying to get string representation of Specifier.";
225 logger.error(result, e);
232 public static class AuthRefConfigInfo {
234 public String getQualifiedDisplayName() {
235 return (Tools.isBlank(schema))
236 ? displayName : DocumentUtils.appendSchemaName(schema, displayName);
239 public String getDisplayName() {
243 public void setDisplayName(String displayName) {
244 this.displayName = displayName;
249 public String getSchema() {
253 public void setSchema(String schema) {
254 this.schema = schema;
257 public String getFullPath() {
261 public void setFullPath(String fullPath) {
262 this.fullPath = fullPath;
265 protected String[] pathEls;
267 public AuthRefConfigInfo(AuthRefConfigInfo arci) {
268 this.displayName = arci.displayName;
269 this.schema = arci.schema;
270 this.fullPath = arci.fullPath;
271 this.pathEls = arci.pathEls;
272 // Skip the pathElse check, since we are creatign from another (presumably valid) arci.
275 public AuthRefConfigInfo(String displayName, String schema, String fullPath, String[] pathEls) {
276 this.displayName = displayName;
277 this.schema = schema;
278 this.fullPath = fullPath;
279 this.pathEls = pathEls;
283 // Split a config value string like "intakes_common:collector", or
284 // "collectionobjects_common:contentPeoples|contentPeople"
285 // "collectionobjects_common:assocEventGroupList/*/assocEventPlace"
286 // If has a pipe ('|') second part is a displayLabel, and first is path
287 // Otherwise, entry is a path, and can use the last pathElement as displayName
288 // Should be schema qualified.
289 public AuthRefConfigInfo(String configString) {
290 String[] pair = configString.split("\\|", 2);
292 String displayName, fullPath;
293 if (pair.length == 1) {
294 // no label specifier, so we'll defer getting label
296 pathEls = pair[0].split("/");
297 displayName = pathEls[pathEls.length - 1];
300 pathEls = pair[0].split("/");
301 displayName = pair[1];
303 String[] schemaSplit = pathEls[0].split(":", 2);
305 if (schemaSplit.length == 1) { // schema not specified
308 schema = schemaSplit[0];
309 if (pair.length == 1 && pathEls.length == 1) { // simplest case of field in top level schema, no labelll
310 displayName = schemaSplit[1]; // Have to fix up displayName to have no schema
313 this.displayName = displayName;
314 this.schema = schema;
315 this.fullPath = fullPath;
316 this.pathEls = pathEls;
320 protected void checkPathEls() {
321 int len = pathEls.length;
323 throw new InternalError("Bad values in authRef info - caller screwed up:" + fullPath);
325 // Handle case of them putting a leading slash on the path
326 if (len > 1 && pathEls[0].endsWith(":")) {
328 String[] newArray = new String[len];
329 newArray[0] = pathEls[0] + pathEls[1];
331 System.arraycopy(pathEls, 2, newArray, 1, len - 1);
338 public static class AuthRefInfo extends AuthRefConfigInfo {
340 public Property getProperty() {
344 public void setProperty(Property property) {
345 this.property = property;
349 public AuthRefInfo(String displayName, String schema, String fullPath, String[] pathEls, Property prop) {
350 super(displayName, schema, fullPath, pathEls);
351 this.property = prop;
354 public AuthRefInfo(AuthRefConfigInfo arci, Property prop) {
356 this.property = prop;
360 private static final Logger logger = LoggerFactory.getLogger(RefNameServiceUtils.class);
361 private static ArrayList<String> refNameServiceTypes = null;
363 public static void updateRefNamesInRelations(
364 ServiceContext<PoxPayloadIn, PoxPayloadOut> ctx,
365 RepositoryClient<PoxPayloadIn, PoxPayloadOut> repoClient,
366 CoreSessionInterface repoSession,
368 String newRefName) throws Exception {
370 // First, look for and update all the places where the refName is the "subject" of the relationship
372 RelationUtils.updateRefNamesInRelations(ctx, repoClient, repoSession, IRelationsManager.SUBJECT_REFNAME, oldRefName, newRefName);
375 // Next, look for and update all the places where the refName is the "object" of the relationship
377 RelationUtils.updateRefNamesInRelations(ctx, repoClient, repoSession, IRelationsManager.OBJECT_REFNAME, oldRefName, newRefName);
380 public static List<AuthRefConfigInfo> getConfiguredAuthorityRefs(ServiceContext<PoxPayloadIn, PoxPayloadOut> ctx) {
381 List<String> authRefFields = ((AbstractServiceContextImpl) ctx).getAllPartsPropertyValues(
382 ServiceBindingUtils.AUTH_REF_PROP, ServiceBindingUtils.QUALIFIED_PROP_NAMES);
383 ArrayList<AuthRefConfigInfo> authRefsInfo = new ArrayList<AuthRefConfigInfo>(authRefFields.size());
384 for (String spec : authRefFields) {
385 AuthRefConfigInfo arci = new AuthRefConfigInfo(spec);
386 authRefsInfo.add(arci);
391 public static AuthorityRefDocList getAuthorityRefDocs(
392 CoreSessionInterface repoSession,
393 ServiceContext<PoxPayloadIn, PoxPayloadOut> ctx,
394 RepositoryClient<PoxPayloadIn, PoxPayloadOut> repoClient,
395 List<String> serviceTypes,
397 String refPropName, // authRef or termRef, authorities or vocab terms.
398 DocumentFilter filter,
399 boolean useDefaultOrderByClause,
400 boolean computeTotal) throws DocumentException, DocumentNotFoundException {
401 AuthorityRefDocList wrapperList = new AuthorityRefDocList();
402 AbstractCommonList commonList = (AbstractCommonList) wrapperList;
403 int pageNum = filter.getStartPage();
404 int pageSize = filter.getPageSize();
406 List<AuthorityRefDocList.AuthorityRefDocItem> list =
407 wrapperList.getAuthorityRefDocItem();
409 Map<String, ServiceBindingType> queriedServiceBindings = new HashMap<String, ServiceBindingType>();
410 Map<String, List<AuthRefConfigInfo>> authRefFieldsByService = new HashMap<String, List<AuthRefConfigInfo>>();
412 NuxeoRepositoryClientImpl nuxeoRepoClient = (NuxeoRepositoryClientImpl) repoClient;
414 // Ignore any provided page size and number query parameters in
415 // the following call, as they pertain to the list of authority
416 // references to be returned, not to the list of documents to be
417 // scanned for those references.
419 // Get a list of possibly referencing documents. This list is
420 // lazily loaded, page by page. Ideally, only one page will
421 // need to be loaded to fill one page of results. Some number
422 // of possibly referencing documents will be false positives,
423 // so use a page size of double the requested page size to
424 // account for those.
425 DocumentModelList docList = findAllAuthorityRefDocs(ctx,
431 queriedServiceBindings,
432 authRefFieldsByService,
433 filter.getWhereClause(),
434 null, // orderByClause
437 useDefaultOrderByClause,
440 if (docList == null) { // found no authRef fields - nothing to process
444 String fieldList = "docType|docId|docNumber|docName|sourceField|uri|refName|updatedAt|workflowState"; // FIXME: Should not be hard-coded string
445 commonList.setFieldsReturned(fieldList);
447 // As a side-effect, the method called below modifies the value of
448 // the 'list' variable, which holds the list of references to
449 // an authority item.
451 // There can be more than one reference to a particular authority
452 // item within any individual document scanned, so the number of
453 // authority references may potentially exceed the total number
454 // of documents scanned.
456 // Strip off displayName and only match the base, so we get references to all
457 // the NPTs as well as the PT.
458 String strippedRefName = RefNameUtils.stripAuthorityTermDisplayName(refName);
460 // *** Need to pass in pagination info here.
461 long nRefsFound = processRefObjsDocListForList(docList, ctx.getTenantId(), strippedRefName,
462 queriedServiceBindings, authRefFieldsByService, // the actual list size needs to be updated to the size of "list"
463 list, pageSize, pageNum);
465 commonList.setPageSize(pageSize);
467 // Values returned in the pagination block above the list items
468 // need to reflect the number of references to authority items
469 // returned, rather than the number of documents originally scanned
470 // to find such references.
471 // This will be an estimate only...
472 commonList.setPageNum(pageNum);
473 commonList.setTotalItems(nRefsFound); // Accurate if total was scanned, otherwise, just an estimate
474 commonList.setItemsInPage(list.size());
476 if (logger.isDebugEnabled() && (nRefsFound < docList.size())) {
477 logger.debug("Internal curiosity: got fewer matches of refs than # docs matched..."); // We found a ref to ourself and have excluded it.
479 } catch (Exception e) {
480 logger.error("Could not retrieve a list of documents referring to the specified authority item", e);
487 private static ArrayList<String> getRefNameServiceTypes() {
488 if (refNameServiceTypes == null) {
489 refNameServiceTypes = new ArrayList<String>();
490 refNameServiceTypes.add(ServiceBindingUtils.SERVICE_TYPE_AUTHORITY);
491 refNameServiceTypes.add(ServiceBindingUtils.SERVICE_TYPE_OBJECT);
492 refNameServiceTypes.add(ServiceBindingUtils.SERVICE_TYPE_PROCEDURE);
494 return refNameServiceTypes;
497 // Seems like a good value - no real data to set this well.
498 // Note: can set this value lower during debugging; e.g. to 3 - ADR 2012-07-10
499 private static final int N_OBJS_TO_UPDATE_PER_LOOP = 100;
501 public static int updateAuthorityRefDocs(
502 ServiceContext<PoxPayloadIn, PoxPayloadOut> ctx,
503 RepositoryClient<PoxPayloadIn, PoxPayloadOut> repoClient,
504 CoreSessionInterface repoSession,
507 String refPropName) throws Exception {
508 Map<String, ServiceBindingType> queriedServiceBindings = new HashMap<String, ServiceBindingType>();
509 Map<String, List<AuthRefConfigInfo>> authRefFieldsByService = new HashMap<String, List<AuthRefConfigInfo>>();
514 int docsInCurrentPage = 0;
515 final String WHERE_CLAUSE_ADDITIONS_VALUE = null;
516 final String ORDER_BY_VALUE = CollectionSpaceClient.CORE_CREATED_AT // "collectionspace_core:createdAt";
517 + ", " + IQueryManager.NUXEO_UUID; // CSPACE-6333: Add secondary sort on uuid, in case records have the same createdAt timestamp.
519 if (repoClient instanceof NuxeoRepositoryClientImpl == false) {
520 throw new InternalError("updateAuthorityRefDocs() called with unknown repoClient type!");
523 try { // REM - How can we deal with transaction and timeout issues here?
524 final int pageSize = N_OBJS_TO_UPDATE_PER_LOOP;
525 DocumentModelList docList;
526 boolean morePages = true;
529 docList = findAuthorityRefDocs(ctx,
532 getRefNameServiceTypes(),
535 queriedServiceBindings,
536 authRefFieldsByService,
537 WHERE_CLAUSE_ADDITIONS_VALUE,
541 true, // useDefaultOrderByClause
542 false); // computeTotal
544 if (docList == null) {
545 logger.debug("updateAuthorityRefDocs: no documents could be found that referenced the old refName");
548 docsInCurrentPage = docList.size();
549 logger.debug("updateAuthorityRefDocs: current page=" + currentPage + " documents included in page=" + docsInCurrentPage);
550 if (docsInCurrentPage == 0) {
551 logger.debug("updateAuthorityRefDocs: no more documents requiring refName updates could be found");
554 if (docsInCurrentPage < pageSize) {
555 logger.debug("updateAuthorityRefDocs: assuming no more documents requiring refName updates will be found, as docsInCurrentPage < pageSize");
559 // Only match complete refNames - unless and until we decide how to resolve changes
560 // to NPTs we will defer that and only change PTs or refNames as passed in.
561 long nRefsFoundThisPage = processRefObjsDocListForUpdate(ctx, docList, ctx.getTenantId(), oldRefName,
562 queriedServiceBindings, authRefFieldsByService, // Perform the refName updates on the list of document models
564 if (nRefsFoundThisPage > 0) {
565 ((NuxeoRepositoryClientImpl) repoClient).saveDocListWithoutHandlerProcessing(ctx, repoSession, docList, true); // Flush the document model list out to Nuxeo storage
566 nRefsFound += nRefsFoundThisPage;
569 // FIXME: Per REM, set a limit of num objects - something like
570 // 1000K objects - and also add a log Warning after some threshold
571 docsScanned += docsInCurrentPage;
577 } catch (Exception e) {
578 logger.error("Internal error updating the AuthorityRefDocs: " + e.getLocalizedMessage());
579 logger.debug(Tools.errorToString(e, true));
582 logger.debug("updateAuthorityRefDocs replaced a total of " + nRefsFound + " authority references, within as many as " + docsScanned + " scanned document(s)");
586 private static DocumentModelList findAllAuthorityRefDocs(
587 ServiceContext<PoxPayloadIn, PoxPayloadOut> ctx,
588 RepositoryClient<PoxPayloadIn, PoxPayloadOut> repoClient,
589 CoreSessionInterface repoSession, List<String> serviceTypes,
592 Map<String, ServiceBindingType> queriedServiceBindings,
593 Map<String, List<AuthRefConfigInfo>> authRefFieldsByService,
594 String whereClauseAdditions,
595 String orderByClause,
598 boolean useDefaultOrderByClause,
599 boolean computeTotal) throws DocumentException, DocumentNotFoundException {
601 return new LazyAuthorityRefDocList(ctx,
607 queriedServiceBindings,
608 authRefFieldsByService,
609 whereClauseAdditions,
612 useDefaultOrderByClause,
616 protected static DocumentModelList findAuthorityRefDocs(
617 ServiceContext<PoxPayloadIn, PoxPayloadOut> ctx,
618 RepositoryClient<PoxPayloadIn, PoxPayloadOut> repoClient,
619 CoreSessionInterface repoSession, List<String> serviceTypes,
622 Map<String, ServiceBindingType> queriedServiceBindings,
623 Map<String, List<AuthRefConfigInfo>> authRefFieldsByService,
624 String whereClauseAdditions,
625 String orderByClause,
628 boolean useDefaultOrderByClause,
629 boolean computeTotal) throws DocumentException, DocumentNotFoundException {
631 // Get the service bindings for this tenant
632 TenantBindingConfigReaderImpl tReader = ServiceMain.getInstance().getTenantBindingConfigReader();
634 // We need to get all the procedures, authorities, and objects.
635 List<ServiceBindingType> servicebindings = tReader.getServiceBindingsByType(ctx.getTenantId(), serviceTypes);
636 if (servicebindings == null || servicebindings.isEmpty()) {
637 logger.error("RefNameServiceUtils.getAuthorityRefDocs: No services bindings found, cannot proceed!");
640 // Filter the list for current user rights
641 servicebindings = SecurityUtils.getReadableServiceBindingsForCurrentUser(servicebindings);
643 ArrayList<String> docTypes = new ArrayList<String>();
645 String query = computeWhereClauseForAuthorityRefDocs(refName, refPropName, docTypes, servicebindings, // REM - Side effect that docTypes, authRefFieldsByService, and queriedServiceBindings get set/change. Any others?
646 queriedServiceBindings, authRefFieldsByService);
647 if (query == null) { // found no authRef fields - nothing to query
651 // Additional qualifications, like workflow state
652 if (Tools.notBlank(whereClauseAdditions)) {
653 query += " AND " + whereClauseAdditions;
656 // Now we have to issue the search
657 DocumentModelList docList = findDocs(
666 useDefaultOrderByClause,
672 private static final DocumentModelList findDocs(
673 RepositoryClient<PoxPayloadIn, PoxPayloadOut> repoClient,
674 ServiceContext<PoxPayloadIn, PoxPayloadOut> ctx,
675 CoreSessionInterface repoSession,
676 List<String> docTypes,
678 String orderByClause,
681 boolean useDefaultOrderByClause,
682 boolean computeTotal)
683 throws DocumentNotFoundException, DocumentException {
684 Profiler profiler = new Profiler(query, 0);
688 UUID uuid = UUID.randomUUID();
690 // Write a CSV-delimited message to the performance log,
691 // in a format intended to be interoperable with those
692 // generated by other system layers.
694 profiler.getStartTime()
695 + "," + uuid.toString()
699 + "," + Thread.currentThread().getName();
700 final boolean FORMAT_LOG_MESSAGE = false;
701 profiler.log(csvMsg, FORMAT_LOG_MESSAGE);
703 // Now we have to issue the search
704 NuxeoRepositoryClientImpl nuxeoRepoClient = (NuxeoRepositoryClientImpl) repoClient;
705 DocumentWrapper<DocumentModelList> docListWrapper = nuxeoRepoClient.findDocs(
713 useDefaultOrderByClause,
715 // Now we gather the info for each document into the list and return
716 DocumentModelList docList = docListWrapper.getWrappedObject();
718 // Stop timing and log performance-related metrics.
722 profiler.getStopTime()
723 + "," + uuid.toString()
727 + "," + Thread.currentThread().getName();
728 profiler.log(csvMsg, FORMAT_LOG_MESSAGE);
735 private static final boolean READY_FOR_COMPLEX_QUERY = true;
737 private static String computeWhereClauseForAuthorityRefDocs(
740 ArrayList<String> docTypes,
741 List<ServiceBindingType> servicebindings,
742 Map<String, ServiceBindingType> queriedServiceBindings,
743 Map<String, List<AuthRefConfigInfo>> authRefFieldsByService) {
745 boolean fFirst = true;
746 List<String> authRefFieldPaths;
747 for (ServiceBindingType sb : servicebindings) {
748 // Gets the property names for each part, qualified with the part label (which
749 // is also the table name, the way that the repository works).
751 ServiceBindingUtils.getAllPartsPropertyValues(sb,
752 refPropName, ServiceBindingUtils.QUALIFIED_PROP_NAMES);
753 if (authRefFieldPaths.isEmpty()) {
756 ArrayList<AuthRefConfigInfo> authRefsInfo = new ArrayList<AuthRefConfigInfo>();
757 for (String spec : authRefFieldPaths) {
758 AuthRefConfigInfo arci = new AuthRefConfigInfo(spec);
759 authRefsInfo.add(arci);
762 String docType = sb.getObject().getName();
763 queriedServiceBindings.put(docType, sb);
764 authRefFieldsByService.put(docType, authRefsInfo);
765 docTypes.add(docType);
768 if (fFirst) { // found no authRef fields - nothing to query
772 // Note that this will also match the term item itself, but that will get filtered out when
773 // we compute actual matches.
774 AuthorityTermInfo authTermInfo = RefNameUtils.parseAuthorityTermInfo(refName);
776 // Example refname: urn:cspace:pahma.cspace.berkeley.edu:personauthorities:name(person):item:name(ReneRichie1586477168934)
777 // Corresponding phrase: "urn cspace pahma cspace berkeley edu personauthorities name person item name ReneRichie1586477168934
779 String refnamePhrase = String.format("urn cspace %s %s name %s item name %s",
780 RefNameUtils.domainToPhrase(authTermInfo.inAuthority.domain),
781 authTermInfo.inAuthority.resource,
782 authTermInfo.inAuthority.name,
785 refnamePhrase = String.format("\"%s\"", refnamePhrase); // surround the phase in double quotes to indicate this is a NXQL phrase search
787 String whereClauseStr = QueryManager.createWhereClauseFromKeywords(refnamePhrase);
789 if (logger.isTraceEnabled()) {
790 logger.trace("The 'where' clause to find refObjs is: ", refnamePhrase);
793 return whereClauseStr;
796 // TODO there are multiple copies of this that should be put somewhere common.
797 protected static String getRefname(DocumentModel docModel) throws ClientException {
798 String result = (String)docModel.getProperty(CollectionSpaceClient.COLLECTIONSPACE_CORE_SCHEMA,
799 CollectionSpaceClient.COLLECTIONSPACE_CORE_REFNAME);
803 private static long processRefObjsDocListForUpdate(
805 DocumentModelList docList,
808 Map<String, ServiceBindingType> queriedServiceBindings,
809 Map<String, List<AuthRefConfigInfo>> authRefFieldsByService,
810 String newAuthorityRefName) {
811 boolean matchBaseOnly = false;
813 if (ctx.shouldForceUpdateRefnameReferences() == true) {
814 refName = RefNameUtils.stripAuthorityTermDisplayName(refName);
815 matchBaseOnly = true;
818 return processRefObjsDocList(docList, tenantId, refName, matchBaseOnly, queriedServiceBindings,
819 authRefFieldsByService, null, 0, 0, newAuthorityRefName);
822 private static long processRefObjsDocListForList(
823 DocumentModelList docList,
826 Map<String, ServiceBindingType> queriedServiceBindings,
827 Map<String, List<AuthRefConfigInfo>> authRefFieldsByService,
828 List<AuthorityRefDocList.AuthorityRefDocItem> list,
829 int pageSize, int pageNum) {
830 return processRefObjsDocList(docList, tenantId, refName, true, queriedServiceBindings,
831 authRefFieldsByService, list, pageSize, pageNum, null);
836 * Runs through the list of found docs, processing them. If list is
837 * non-null, then processing means gather the info for items. If list is
838 * null, and newRefName is non-null, then processing means replacing and
839 * updating. If processing/updating, this must be called in the context of
840 * an open session, and caller must release Session after calling this.
843 private static long processRefObjsDocList(
844 DocumentModelList docList,
847 boolean matchBaseOnly,
848 Map<String, ServiceBindingType> queriedServiceBindings,
849 Map<String, List<AuthRefConfigInfo>> authRefFieldsByService,
850 List<AuthorityRefDocList.AuthorityRefDocItem> list,
851 int pageSize, int pageNum, // Only used when constructing a list.
852 String newAuthorityRefName) {
853 UriTemplateRegistry registry = ServiceMain.getInstance().getUriTemplateRegistry();
854 Iterator<DocumentModel> iter = docList.iterator();
855 long nRefsFoundTotal = 0;
856 long nRefsFalsePositives = 0;
857 boolean foundSelf = false;
858 boolean warningLogged = false;
860 // When paginating results, we have to guess at the total. First guess is the number of docs returned
861 // by the query. However, this returns some false positives, so may be high.
862 // In addition, we can match multiple fields per doc, so this may be low. Fun, eh?
863 long nDocsReturnedInQuery = (int)docList.totalSize();
864 long nDocsProcessed = 0;
865 long firstItemInPage = pageNum*pageSize;
866 while (iter.hasNext()) {
867 if (!warningLogged && (float)nRefsFalsePositives / nDocsReturnedInQuery > 0.5) {
868 warningLogged = true;
869 String msg = String.format("When searching for documents referencing the term '%s', more than 1/2 of the results were false-positives.",
873 DocumentModel docModel = iter.next();
874 AuthorityRefDocList.AuthorityRefDocItem ilistItem;
876 String docType = docModel.getDocumentType().getName(); // REM - This will be a tentant qualified document type
877 docType = ServiceBindingUtils.getUnqualifiedTenantDocType(docType);
878 ServiceBindingType sb = queriedServiceBindings.get(docType);
880 throw new RuntimeException(
881 "getAuthorityRefDocs: No Service Binding for docType: " + docType);
884 if (list == null) { // no list - should be update refName case.
885 if (newAuthorityRefName == null) {
886 throw new InternalError("processRefObjsDocList() called with neither an itemList nor a new RefName!");
890 firstItemInPage = 0; // Do not paginate if updating, rather than building list
891 } else { // Have a list - refObjs case
892 if (newAuthorityRefName != null) {
893 throw new InternalError("processRefObjsDocList() called with both an itemList and a new RefName!");
895 if (firstItemInPage > 100) {
896 String msg = String.format("Processing a large offset for records referencing (term:%s, size:%d, num:%d) - will be expensive!!!",
897 refName, pageSize, pageNum);
900 // Note that we have to go through check all the fields to determine the actual page start
901 ilistItem = new AuthorityRefDocList.AuthorityRefDocItem();
902 String csid = NuxeoUtils.getCsid(docModel);//NuxeoUtils.extractId(docModel.getPathAsString());
904 String itemRefName = getRefname(docModel);
905 ilistItem.setRefName(itemRefName);
906 } catch (ClientException ce) {
907 throw new RuntimeException(
908 "processRefObjsDocList: Problem fetching refName from item Object: "
909 + ce.getLocalizedMessage());
911 ilistItem.setDocId(csid);
913 UriTemplateRegistryKey key = new UriTemplateRegistryKey(tenantId, docType);
914 StoredValuesUriTemplate template = registry.get(key);
915 if (template != null) {
916 Map<String, String> additionalValues = new HashMap<String, String>();
917 if (template.getUriTemplateType() == UriTemplateFactory.RESOURCE) {
918 additionalValues.put(UriTemplateFactory.IDENTIFIER_VAR, csid);
919 uri = template.buildUri(additionalValues);
920 } else if (template.getUriTemplateType() == UriTemplateFactory.ITEM) {
922 String inAuthorityCsid = (String) NuxeoUtils.getProperyValue(docModel, "inAuthority"); //docModel.getPropertyValue("inAuthority"); // AuthorityItemJAXBSchema.IN_AUTHORITY
923 additionalValues.put(UriTemplateFactory.IDENTIFIER_VAR, inAuthorityCsid);
924 additionalValues.put(UriTemplateFactory.ITEM_IDENTIFIER_VAR, csid);
925 uri = template.buildUri(additionalValues);
926 } catch (Exception e) {
927 logger.warn("Could not extract inAuthority property from authority item record: " + e.getMessage());
929 } else if (template.getUriTemplateType() == UriTemplateFactory.CONTACT) {
930 // FIXME: Generating contact sub-resource URIs requires additional work,
931 // as a follow-on to CSPACE-5271 - ADR 2012-08-16
932 // Sets the default (empty string) value for uri, for now
934 logger.warn("Unrecognized URI template type = " + template.getUriTemplateType());
935 // Sets the default (empty string) value for uri
937 } else { // (if template == null)
938 logger.warn("Could not retrieve URI template from registry via tenant ID "
939 + tenantId + " and docType " + docType);
940 // Sets the default (empty string) value for uri
942 ilistItem.setUri(uri);
944 ilistItem.setWorkflowState(docModel.getCurrentLifeCycleState());
945 ilistItem.setUpdatedAt(NuxeoDocumentModelHandler.getUpdatedAtAsString(docModel));
946 } catch (Exception e) {
947 logger.error("Error getting core values for doc [" + csid + "]: " + e.getLocalizedMessage());
949 ilistItem.setDocType(docType);
950 ilistItem.setDocNumber(
951 ServiceBindingUtils.getMappedFieldInDoc(sb, ServiceBindingUtils.OBJ_NUMBER_PROP, docModel));
952 ilistItem.setDocName(
953 ServiceBindingUtils.getMappedFieldInDoc(sb, ServiceBindingUtils.OBJ_NAME_PROP, docModel));
955 // Now, we have to loop over the authRefFieldsByService to figure out
956 // out which field(s) matched this.
957 List<AuthRefConfigInfo> matchingAuthRefFields = authRefFieldsByService.get(docType);
958 if (matchingAuthRefFields == null || matchingAuthRefFields.isEmpty()) {
959 throw new RuntimeException(
960 "getAuthorityRefDocs: internal logic error: can't fetch authRefFields for DocType.");
963 ArrayList<RefNameServiceUtils.AuthRefInfo> foundProps = new ArrayList<RefNameServiceUtils.AuthRefInfo>();
965 findAuthRefPropertiesInDoc(docModel, matchingAuthRefFields, refName, matchBaseOnly, foundProps); // REM - side effect that foundProps is set
966 if(!foundProps.isEmpty()) {
967 int nRefsFoundInDoc = 0;
968 for (RefNameServiceUtils.AuthRefInfo ari : foundProps) {
969 if (ilistItem != null) {
970 // So this is a true positive, and not a false one. We have to consider pagination now.
971 if(nRefsFoundTotal >= firstItemInPage) { // skipped enough already
972 if (nRefsFoundInDoc == 0) { // First one?
973 ilistItem.setSourceField(ari.getQualifiedDisplayName());
974 ilistItem.setValue((String) ari.getProperty().getValue());
975 } else { // duplicates from one object
976 ilistItem = cloneAuthRefDocItem(ilistItem, ari.getQualifiedDisplayName(), (String) ari.getProperty().getValue());
979 nRefsFoundInDoc++; // Only increment if processed, or clone logic above will fail
981 } else { // update refName case
982 Property propToUpdate = ari.getProperty();
983 propToUpdate.setValue(newAuthorityRefName);
985 nRefsFoundTotal++; // Whether we processed or not, we found - essential to pagination logic
987 } else if(ilistItem != null) {
988 String docRefName = ilistItem.getRefName();
990 (docRefName!=null && docRefName.startsWith(refName))
991 :refName.equals(docRefName)) {
992 // We found the self for an item
994 logger.trace("getAuthorityRefDocs: Result: "
995 + docType + " [" + NuxeoUtils.getCsid(docModel)
996 + "] appears to be self for: ["
999 nRefsFalsePositives++;
1000 logger.trace("getAuthorityRefDocs: Result: "
1001 + docType + " [" + NuxeoUtils.getCsid(docModel)
1002 + "] does not reference ["
1006 } catch (ClientException ce) {
1007 throw new RuntimeException(
1008 "getAuthorityRefDocs: Problem fetching values from repo: " + ce.getLocalizedMessage());
1013 // Done processing that doc. Are we done with the whole page?
1014 // Note pageSize <=0 means do them all
1015 if ((pageSize > 0) && ((nRefsFoundTotal - firstItemInPage) >= pageSize)) {
1016 // Quitting early, so we need to estimate the total. Assume one per doc
1017 // for the rest of the docs we matched in the query
1018 long unprocessedDocs = nDocsReturnedInQuery - nDocsProcessed;
1019 if (unprocessedDocs > 0) {
1020 // We generally match ourselves in the keyword search. If we already saw ourselves
1021 // then do not try to correct for this. Otherwise, decrement the total.
1022 // Yes, this is fairly goofy, but the whole estimation mechanism is goofy.
1025 nRefsFoundTotal += unprocessedDocs;
1029 } // close while(iterator)
1031 // Log a final warning if we find too many false-positives.
1032 if ((float)nRefsFalsePositives / nDocsReturnedInQuery > 0.33) {
1033 String msg = String.format("Found %d false-positives and %d only true references the refname:%s",
1034 nRefsFalsePositives, nRefsFoundTotal, refName);
1038 return nRefsFoundTotal;
1042 * Clone an AuthorityRefDocItem which is a JAX-B generated class. Be sure we're copying every field defined in the XSD (XML Schema) that is
1043 * found here services\jaxb\src\main\resources\authorityrefdocs.xsd
1045 * @param sourceField
1048 private static AuthorityRefDocList.AuthorityRefDocItem cloneAuthRefDocItem(
1049 AuthorityRefDocList.AuthorityRefDocItem ilistItem, String sourceField, String value) {
1050 AuthorityRefDocList.AuthorityRefDocItem newlistItem = new AuthorityRefDocList.AuthorityRefDocItem();
1051 newlistItem.setDocId(ilistItem.getDocId());
1052 newlistItem.setDocName(ilistItem.getDocName());
1053 newlistItem.setDocNumber(ilistItem.getDocNumber());
1054 newlistItem.setDocType(ilistItem.getDocType());
1055 newlistItem.setUri(ilistItem.getUri());
1056 newlistItem.setSourceField(sourceField);
1057 newlistItem.setValue(value);
1058 newlistItem.setRefName(ilistItem.getRefName());
1059 newlistItem.setUpdatedAt(ilistItem.getUpdatedAt());
1060 newlistItem.setWorkflowState(ilistItem.getWorkflowState());
1064 public static List<AuthRefInfo> findAuthRefPropertiesInDoc(
1065 DocumentModel docModel,
1066 List<AuthRefConfigInfo> authRefFieldInfoList,
1067 String refNameToMatch,
1068 List<AuthRefInfo> foundProps) {
1069 return findAuthRefPropertiesInDoc(docModel, authRefFieldInfoList,
1070 refNameToMatch, false, foundProps);
1073 public static List<AuthRefInfo> findAuthRefPropertiesInDoc(
1074 DocumentModel docModel,
1075 List<AuthRefConfigInfo> authRefFieldInfoList,
1076 String refNameToMatch,
1077 boolean matchBaseOnly,
1078 List<AuthRefInfo> authRefInfoList) {
1079 // Assume that authRefFieldInfo is keyed by the field name (possibly mapped for UI)
1080 // and the values are elPaths to the field, where intervening group structures in
1081 // lists of complex structures are replaced with "*". Thus, valid paths include
1082 // the following (note that the ServiceBindingUtils prepend schema names to configured values):
1083 // "schemaname:fieldname"
1084 // "schemaname:scalarlistname"
1085 // "schemaname:complexfieldname/fieldname"
1086 // "schemaname:complexlistname/*/fieldname"
1087 // "schemaname:complexlistname/*/scalarlistname"
1088 // "schemaname:complexlistname/*/complexfieldname/fieldname"
1089 // "schemaname:complexlistname/*/complexlistname/*/fieldname"
1091 for (AuthRefConfigInfo arci : authRefFieldInfoList) {
1093 // Get first property and work down as needed.
1094 Property prop = docModel.getProperty(arci.pathEls[0]);
1095 findAuthRefPropertiesInProperty(authRefInfoList, prop, arci, 0, refNameToMatch, matchBaseOnly);
1096 } catch (Exception e) {
1097 logger.error("Problem fetching property: " + arci.pathEls[0]);
1100 return authRefInfoList;
1103 private static List<AuthRefInfo> findAuthRefPropertiesInProperty(
1104 List<AuthRefInfo> authRefInfoList,
1106 AuthRefConfigInfo arci,
1107 int pathStartIndex, // Supports recursion and we work down the path
1108 String refNameToMatch,
1109 boolean matchBaseOnly ) {
1110 if (pathStartIndex >= arci.pathEls.length) {
1111 throw new ArrayIndexOutOfBoundsException("Index = " + pathStartIndex + " for path: "
1112 + arci.pathEls.toString());
1114 AuthRefInfo ari = null;
1116 return authRefInfoList;
1119 if (prop instanceof StringProperty) { // scalar string
1120 addARIifMatches(refNameToMatch, matchBaseOnly, arci, prop, authRefInfoList); // REM - Side effect that foundProps gets changed/updated
1121 } else if (prop instanceof List) {
1122 List<Property> propList = (List<Property>) prop;
1123 // run through list. Must either be list of Strings, or Complex
1124 for (Property listItemProp : propList) {
1125 if (listItemProp instanceof StringProperty) {
1126 if (arci.pathEls.length - pathStartIndex != 1) {
1127 logger.error("Configuration for authRefs does not match schema structure: "
1128 + arci.pathEls.toString());
1131 addARIifMatches(refNameToMatch, matchBaseOnly, arci, listItemProp, authRefInfoList);
1133 } else if (listItemProp.isComplex()) {
1134 // Just recurse to handle this. Note that since this is a list of complex,
1135 // which should look like listName/*/... we add 2 to the path start index
1136 findAuthRefPropertiesInProperty(authRefInfoList, listItemProp, arci,
1137 pathStartIndex + 2, refNameToMatch, matchBaseOnly);
1139 logger.error("Configuration for authRefs does not match schema structure: "
1140 + arci.pathEls.toString());
1144 } else if (prop.isComplex()) {
1145 String localPropName = arci.pathEls[pathStartIndex];
1147 Property localProp = prop.get(localPropName);
1148 // Now just recurse, pushing down the path 1 step
1149 findAuthRefPropertiesInProperty(authRefInfoList, localProp, arci,
1150 pathStartIndex, refNameToMatch, matchBaseOnly);
1151 } catch (PropertyNotFoundException pnfe) {
1152 logger.error("Could not find property: [" + localPropName + "] in path: "
1153 + arci.getFullPath());
1154 // Fall through - ari will be null and we will continue...
1157 logger.error("Configuration for authRefs does not match schema structure: "
1158 + arci.pathEls.toString());
1162 authRefInfoList.add(ari); //FIXME: REM - This is dead code. 'ari' is never touched after being initalized to null. Why?
1165 return authRefInfoList;
1168 private static void addARIifMatches(
1169 String refNameToMatch,
1170 boolean matchBaseOnly,
1171 AuthRefConfigInfo arci,
1173 List<AuthRefInfo> authRefInfoList) {
1174 // Need to either match a passed refName
1175 // OR have no refName to match but be non-empty
1177 String value = (String) prop.getValue();
1178 if (((refNameToMatch != null) &&
1180 (value!=null && value.startsWith(refNameToMatch))
1181 :refNameToMatch.equals(value)))
1182 || ((refNameToMatch == null) && Tools.notBlank(value))) {
1184 logger.debug("Found a match on property: " + prop.getPath() + " with value: [" + value + "]");
1185 AuthRefInfo ari = new AuthRefInfo(arci, prop);
1186 authRefInfoList.add(ari);
1188 } catch (PropertyException pe) {
1189 logger.debug("PropertyException on: " + prop.getPath() + pe.getLocalizedMessage());
1193 public static String buildWhereForAuthByName(String authorityCommonSchemaName, String name) {
1194 return authorityCommonSchemaName
1195 + ":" + AuthorityJAXBSchema.SHORT_IDENTIFIER
1196 + "='" + name + "'";
1200 * Build an NXQL query for finding an item by its short ID
1202 * @param authorityItemCommonSchemaName
1207 public static String buildWhereForAuthItemByName(String authorityItemCommonSchemaName, String shortId, String parentcsid) {
1208 String result = null;
1210 result = String.format("%s:%s='%s'", authorityItemCommonSchemaName, AuthorityItemJAXBSchema.SHORT_IDENTIFIER, shortId);
1212 // Technically, we don't need the parent CSID since the short ID is unique so it can be null
1214 if (parentcsid != null) {
1215 result = String.format("%s AND %s:%s='%s'",
1216 result, authorityItemCommonSchemaName, AuthorityItemJAXBSchema.IN_AUTHORITY, parentcsid);
1223 * Identifies whether the refName was found in the supplied field. If passed
1224 * a new RefName, will set that into fields in which the old one was found.
1226 * Only works for: * Scalar fields * Repeatable scalar fields (aka
1227 * multi-valued fields)
1229 * Does not work for: * Structured fields (complexTypes) * Repeatable
1230 * structured fields (repeatable complexTypes) private static int
1231 * refNameFoundInField(String oldRefName, Property fieldValue, String
1232 * newRefName) { int nFound = 0; if (fieldValue instanceof List) {
1233 * List<Property> fieldValueList = (List) fieldValue; for (Property
1234 * listItemValue : fieldValueList) { try { if ((listItemValue instanceof
1235 * StringProperty) &&
1236 * oldRefName.equalsIgnoreCase((String)listItemValue.getValue())) {
1237 * nFound++; if(newRefName!=null) { fieldValue.setValue(newRefName); } else
1238 * { // We cannot quit after the first, if we are replacing values. // If we
1239 * are just looking (not replacing), finding one is enough. break; } } }
1240 * catch( PropertyException pe ) {} } } else { try { if ((fieldValue
1241 * instanceof StringProperty) &&
1242 * oldRefName.equalsIgnoreCase((String)fieldValue.getValue())) { nFound++;
1243 * if(newRefName!=null) { fieldValue.setValue(newRefName); } } } catch(
1244 * PropertyException pe ) {} } return nFound; }