git.aero2k.de Git - tmp/jakarta-migration.git/blob

   1 /**
   2  * This document is a part of the source code and related artifacts for
   3  * CollectionSpace, an open source collections management system for museums and
   4  * related institutions:
   5  *
   6  * http://www.collectionspace.org http://wiki.collectionspace.org
   7  *
   8  * Copyright 2009 University of California at Berkeley
   9  *
  10  * Licensed under the Educational Community License (ECL), Version 2.0. You may
  11  * not use this file except in compliance with this License.
  12  *
  13  * You may obtain a copy of the ECL 2.0 License at
  14  *
  15  * https://source.collectionspace.org/collection-space/LICENSE.txt
  16  *
  17  * Unless required by applicable law or agreed to in writing, software
  18  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  19  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  20  * License for the specific language governing permissions and limitations under
  21  * the License.
  22  */
  23 package org.collectionspace.services.common.vocabulary;
  24
  25 import java.util.ArrayList;
  26 import java.util.HashMap;
  27 import java.util.Iterator;
  28 import java.util.List;
  29 import java.util.Map;
  30
  31 import org.nuxeo.ecm.core.api.ClientException;
  32 import org.nuxeo.ecm.core.api.DocumentModel;
  33 import org.nuxeo.ecm.core.api.DocumentModelList;
  34 import org.nuxeo.ecm.core.api.model.Property;
  35 import org.nuxeo.ecm.core.api.model.PropertyException;
  36 import org.nuxeo.ecm.core.api.model.PropertyNotFoundException;
  37 import org.nuxeo.ecm.core.api.model.impl.primitives.StringProperty;
  38 import org.nuxeo.ecm.core.api.CoreSession;
  39
  40 import org.slf4j.Logger;
  41 import org.slf4j.LoggerFactory;
  42
  43 import org.collectionspace.services.client.CollectionSpaceClient;
  44 import org.collectionspace.services.client.IQueryManager;
  45 import org.collectionspace.services.client.IRelationsManager;
  46 import org.collectionspace.services.client.PoxPayloadIn;
  47 import org.collectionspace.services.client.PoxPayloadOut;
  48 import org.collectionspace.services.common.ServiceMain;
  49 import org.collectionspace.services.common.StoredValuesUriTemplate;
  50 import org.collectionspace.services.common.UriTemplateFactory;
  51 import org.collectionspace.services.common.UriTemplateRegistry;
  52 import org.collectionspace.services.common.UriTemplateRegistryKey;
  53 import org.collectionspace.services.common.context.ServiceContext;
  54 import org.collectionspace.services.common.context.AbstractServiceContextImpl;
  55 import org.collectionspace.services.common.api.RefNameUtils;
  56 import org.collectionspace.services.common.api.Tools;
  57 import org.collectionspace.services.common.api.RefNameUtils.AuthorityTermInfo;
  58 import org.collectionspace.services.common.authorityref.AuthorityRefDocList;
  59 import org.collectionspace.services.common.config.TenantBindingConfigReaderImpl;
  60 import org.collectionspace.services.common.context.ServiceBindingUtils;
  61 import org.collectionspace.services.common.document.DocumentException;
  62 import org.collectionspace.services.common.document.DocumentFilter;
  63 import org.collectionspace.services.common.document.DocumentNotFoundException;
  64 import org.collectionspace.services.common.document.DocumentUtils;
  65 import org.collectionspace.services.common.document.DocumentWrapper;
  66 import org.collectionspace.services.common.query.QueryManager;
  67 import org.collectionspace.services.common.relation.RelationUtils;
  68 import org.collectionspace.services.common.repository.RepositoryClient;
  69 import org.collectionspace.services.nuxeo.client.java.CoreSessionInterface;
  70 import org.collectionspace.services.nuxeo.client.java.NuxeoDocumentModelHandler;
  71 import org.collectionspace.services.nuxeo.client.java.RepositoryJavaClientImpl;
  72 import org.collectionspace.services.common.security.SecurityUtils;
  73 import org.collectionspace.services.config.service.ServiceBindingType;
  74 import org.collectionspace.services.jaxb.AbstractCommonList;
  75 import org.collectionspace.services.nuxeo.util.NuxeoUtils;
  76
  77 /**
  78  * RefNameServiceUtils is a collection of services utilities related to refName
  79  * usage.
  80  *
  81  * $LastChangedRevision: $ $LastChangedDate: $
  82  */
  83 public class RefNameServiceUtils {
  84
  85     public static class AuthRefConfigInfo {
  86
  87         public String getQualifiedDisplayName() {
  88             return (Tools.isBlank(schema))
  89                     ? displayName : DocumentUtils.appendSchemaName(schema, displayName);
  90         }
  91
  92         public String getDisplayName() {
  93             return displayName;
  94         }
  95
  96         public void setDisplayName(String displayName) {
  97             this.displayName = displayName;
  98         }
  99         String displayName;
 100         String schema;
 101
 102         public String getSchema() {
 103             return schema;
 104         }
 105
 106         public void setSchema(String schema) {
 107             this.schema = schema;
 108         }
 109
 110         public String getFullPath() {
 111             return fullPath;
 112         }
 113
 114         public void setFullPath(String fullPath) {
 115             this.fullPath = fullPath;
 116         }
 117         String fullPath;
 118         protected String[] pathEls;
 119
 120         public AuthRefConfigInfo(AuthRefConfigInfo arci) {
 121             this.displayName = arci.displayName;
 122             this.schema = arci.schema;
 123             this.fullPath = arci.fullPath;
 124             this.pathEls = arci.pathEls;
 125             // Skip the pathElse check, since we are creatign from another (presumably valid) arci.
 126         }
 127
 128         public AuthRefConfigInfo(String displayName, String schema, String fullPath, String[] pathEls) {
 129             this.displayName = displayName;
 130             this.schema = schema;
 131             this.fullPath = fullPath;
 132             this.pathEls = pathEls;
 133             checkPathEls();
 134         }
 135
 136         // Split a config value string like "intakes_common:collector", or
 137         // "collectionobjects_common:contentPeoples|contentPeople"
 138         // "collectionobjects_common:assocEventGroupList/*/assocEventPlace"
 139         // If has a pipe ('|') second part is a displayLabel, and first is path
 140         // Otherwise, entry is a path, and can use the last pathElement as displayName
 141         // Should be schema qualified.
 142         public AuthRefConfigInfo(String configString) {
 143             String[] pair = configString.split("\\|", 2);
 144             String[] pathEls;
 145             String displayName, fullPath;
 146             if (pair.length == 1) {
 147                 // no label specifier, so we'll defer getting label
 148                 fullPath = pair[0];
 149                 pathEls = pair[0].split("/");
 150                 displayName = pathEls[pathEls.length - 1];
 151             } else {
 152                 fullPath = pair[0];
 153                 pathEls = pair[0].split("/");
 154                 displayName = pair[1];
 155             }
 156             String[] schemaSplit = pathEls[0].split(":", 2);
 157             String schema;
 158             if (schemaSplit.length == 1) {    // schema not specified
 159                 schema = null;
 160             } else {
 161                 schema = schemaSplit[0];
 162                 if (pair.length == 1 && pathEls.length == 1) {    // simplest case of field in top level schema, no labelll
 163                     displayName = schemaSplit[1];    // Have to fix up displayName to have no schema
 164                 }
 165             }
 166             this.displayName = displayName;
 167             this.schema = schema;
 168             this.fullPath = fullPath;
 169             this.pathEls = pathEls;
 170             checkPathEls();
 171         }
 172
 173         protected void checkPathEls() {
 174             int len = pathEls.length;
 175             if (len < 1) {
 176                 throw new InternalError("Bad values in authRef info - caller screwed up:" + fullPath);
 177             }
 178             // Handle case of them putting a leading slash on the path
 179             if (len > 1 && pathEls[0].endsWith(":")) {
 180                 len--;
 181                 String[] newArray = new String[len];
 182                 newArray[0] = pathEls[0] + pathEls[1];
 183                 if (len >= 2) {
 184                     System.arraycopy(pathEls, 2, newArray, 1, len - 1);
 185                 }
 186                 pathEls = newArray;
 187             }
 188         }
 189     }
 190
 191     public static class AuthRefInfo extends AuthRefConfigInfo {
 192
 193         public Property getProperty() {
 194             return property;
 195         }
 196
 197         public void setProperty(Property property) {
 198             this.property = property;
 199         }
 200         Property property;
 201
 202         public AuthRefInfo(String displayName, String schema, String fullPath, String[] pathEls, Property prop) {
 203             super(displayName, schema, fullPath, pathEls);
 204             this.property = prop;
 205         }
 206
 207         public AuthRefInfo(AuthRefConfigInfo arci, Property prop) {
 208             super(arci);
 209             this.property = prop;
 210         }
 211     }
 212
 213     private static final Logger logger = LoggerFactory.getLogger(RefNameServiceUtils.class);
 214     private static ArrayList<String> refNameServiceTypes = null;
 215
 216     public static void updateRefNamesInRelations(
 217             ServiceContext<PoxPayloadIn, PoxPayloadOut> ctx,
 218             RepositoryClient<PoxPayloadIn, PoxPayloadOut> repoClient,
 219             CoreSessionInterface repoSession,
 220             String oldRefName,
 221             String newRefName) throws Exception {
 222         //
 223         // First, look for and update all the places where the refName is the "subject" of the relationship
 224         //
 225         RelationUtils.updateRefNamesInRelations(ctx, repoClient, repoSession, IRelationsManager.SUBJECT_REFNAME, oldRefName, newRefName);
 226
 227         //
 228         // Next, look for and update all the places where the refName is the "object" of the relationship
 229         //
 230         RelationUtils.updateRefNamesInRelations(ctx, repoClient, repoSession, IRelationsManager.OBJECT_REFNAME, oldRefName, newRefName);
 231     }
 232
 233         public static List<AuthRefConfigInfo> getConfiguredAuthorityRefs(ServiceContext<PoxPayloadIn, PoxPayloadOut> ctx) {
 234                 List<String> authRefFields = ((AbstractServiceContextImpl) ctx).getAllPartsPropertyValues(
 235                                 ServiceBindingUtils.AUTH_REF_PROP, ServiceBindingUtils.QUALIFIED_PROP_NAMES);
 236                 ArrayList<AuthRefConfigInfo> authRefsInfo = new ArrayList<AuthRefConfigInfo>(authRefFields.size());
 237                 for (String spec : authRefFields) {
 238                         AuthRefConfigInfo arci = new AuthRefConfigInfo(spec);
 239                         authRefsInfo.add(arci);
 240                 }
 241                 return authRefsInfo;
 242         }
 243
 244     public static AuthorityRefDocList getAuthorityRefDocs(
 245                 CoreSessionInterface repoSession,
 246             ServiceContext<PoxPayloadIn, PoxPayloadOut> ctx,
 247             UriTemplateRegistry uriTemplateRegistry,
 248             RepositoryClient<PoxPayloadIn, PoxPayloadOut> repoClient,
 249             List<String> serviceTypes,
 250             String refName,
 251             String refPropName, // authRef or termRef, authorities or vocab terms.
 252             DocumentFilter filter, boolean computeTotal)
 253             throws DocumentException, DocumentNotFoundException {
 254         AuthorityRefDocList wrapperList = new AuthorityRefDocList();
 255         AbstractCommonList commonList = (AbstractCommonList) wrapperList;
 256         int pageNum = filter.getStartPage();
 257         int pageSize = filter.getPageSize();
 258
 259         List<AuthorityRefDocList.AuthorityRefDocItem> list =
 260                 wrapperList.getAuthorityRefDocItem();
 261
 262         Map<String, ServiceBindingType> queriedServiceBindings = new HashMap<String, ServiceBindingType>();
 263         Map<String, List<AuthRefConfigInfo>> authRefFieldsByService = new HashMap<String, List<AuthRefConfigInfo>>();
 264
 265         RepositoryJavaClientImpl nuxeoRepoClient = (RepositoryJavaClientImpl) repoClient;
 266         try {
 267             // Ignore any provided page size and number query parameters in
 268             // the following call, as they pertain to the list of authority
 269             // references to be returned, not to the list of documents to be
 270             // scanned for those references.
 271
 272             // Get a list of possibly referencing documents. This list is
 273             // lazily loaded, page by page. Ideally, only one page will
 274             // need to be loaded to fill one page of results. Some number
 275             // of possibly referencing documents will be false positives,
 276             // so use a page size of double the requested page size to
 277             // account for those.
 278             DocumentModelList docList = findAllAuthorityRefDocs(ctx, repoClient, repoSession,
 279                     serviceTypes, refName, refPropName, queriedServiceBindings, authRefFieldsByService,
 280                     filter.getWhereClause(), null, 2*pageSize, computeTotal);
 281
 282             if (docList == null) { // found no authRef fields - nothing to process
 283                 return wrapperList;
 284             }
 285
 286             // set the fieldsReturned list. Even though this is a fixed schema, app layer treats
 287             // this like other abstract common lists
 288             /*
 289              * <xs:element name="docType" type="xs:string" minOccurs="1" />
 290              * <xs:element name="docId" type="xs:string" minOccurs="1" />
 291              * <xs:element name="docNumber" type="xs:string" minOccurs="0" />
 292              * <xs:element name="docName" type="xs:string" minOccurs="0" />
 293              * <xs:element name="sourceField" type="xs:string" minOccurs="1" />
 294              * <xs:element name="uri" type="xs:anyURI" minOccurs="1" />
 295              * <xs:element name="refName" type="xs:String" minOccurs="1" />
 296              * <xs:element name="updatedAt" type="xs:string" minOccurs="1" />
 297              * <xs:element name="workflowState" type="xs:string" minOccurs="1"
 298              * />
 299              */
 300             String fieldList = "docType|docId|docNumber|docName|sourceField|uri|refName|updatedAt|workflowState";
 301             commonList.setFieldsReturned(fieldList);
 302
 303             // As a side-effect, the method called below modifies the value of
 304             // the 'list' variable, which holds the list of references to
 305             // an authority item.
 306             //
 307             // There can be more than one reference to a particular authority
 308             // item within any individual document scanned, so the number of
 309             // authority references may potentially exceed the total number
 310             // of documents scanned.
 311
 312             // Strip off displayName and only match the base, so we get references to all
 313             // the NPTs as well as the PT.
 314                 String strippedRefName = RefNameUtils.stripAuthorityTermDisplayName(refName);
 315
 316                 // *** Need to pass in pagination info here.
 317             int nRefsFound = processRefObjsDocListForList(docList, ctx.getTenantId(), strippedRefName,
 318                         queriedServiceBindings, authRefFieldsByService, // the actual list size needs to be updated to the size of "list"
 319                     list, pageSize, pageNum);
 320
 321             commonList.setPageSize(pageSize);
 322
 323             // Values returned in the pagination block above the list items
 324             // need to reflect the number of references to authority items
 325             // returned, rather than the number of documents originally scanned
 326             // to find such references.
 327             // This will be an estimate only...
 328             commonList.setPageNum(pageNum);
 329                 commonList.setTotalItems(nRefsFound);   // Accurate if total was scanned, otherwise, just an estimate
 330             commonList.setItemsInPage(list.size());
 331
 332             /* Pagination is now handled in the processing step
 333             // Slice the list to return only the specified page of items
 334             // in the list results.
 335             //
 336             // FIXME: There may well be a pattern-based way to do this
 337             // in our framework, and if we can eliminate much of the
 338             // non-DRY code below, that would be desirable.
 339
 340             int startIndex = 0;
 341             int endIndex = 0;
 342
 343             // Return all results if pageSize is 0.
 344             if (pageSize == 0) {
 345                 startIndex = 0;
 346                 endIndex = list.size();
 347             } else {
 348                startIndex = pageNum * pageSize;
 349             }
 350
 351             // Return an empty list when the start of the requested page is
 352             // beyond the last item in the list.
 353             if (startIndex > list.size()) {
 354                 wrapperList.getAuthorityRefDocItem().clear();
 355                 commonList.setItemsInPage(wrapperList.getAuthorityRefDocItem().size());
 356                 return wrapperList;
 357             }
 358
 359             // Otherwise, return a list of items from the start of the specified
 360             // page through the last item on that page, or otherwise through the
 361             // last item in the entire list, if that occurs earlier than the end
 362             // of the specified page.
 363             if (endIndex == 0) {
 364                 int pageEndIndex = ((startIndex + pageSize));
 365                 endIndex = (pageEndIndex > list.size()) ? list.size() : pageEndIndex;
 366             }
 367
 368             // Slice the list to return only the specified page of results.
 369             // Note: the second argument to List.subList(), endIndex, is
 370             // exclusive of the item at its index position, reflecting the
 371             // zero-index nature of the list.
 372             List<AuthorityRefDocList.AuthorityRefDocItem> currentPageList =
 373                     new ArrayList<AuthorityRefDocList.AuthorityRefDocItem>(list.subList(startIndex, endIndex));
 374             wrapperList.getAuthorityRefDocItem().clear();
 375             wrapperList.getAuthorityRefDocItem().addAll(currentPageList);
 376             commonList.setItemsInPage(currentPageList.size());
 377             */
 378
 379             if (logger.isDebugEnabled() && (nRefsFound < docList.size())) {
 380                 logger.debug("Internal curiosity: got fewer matches of refs than # docs matched..."); // We found a ref to ourself and have excluded it.
 381             }
 382         } catch (Exception e) {
 383             logger.error("Could not retrieve a list of documents referring to the specified authority item", e);
 384             wrapperList = null;
 385         }
 386
 387         return wrapperList;
 388     }
 389
 390     private static ArrayList<String> getRefNameServiceTypes() {
 391         if (refNameServiceTypes == null) {
 392             refNameServiceTypes = new ArrayList<String>();
 393             refNameServiceTypes.add(ServiceBindingUtils.SERVICE_TYPE_AUTHORITY);
 394             refNameServiceTypes.add(ServiceBindingUtils.SERVICE_TYPE_OBJECT);
 395             refNameServiceTypes.add(ServiceBindingUtils.SERVICE_TYPE_PROCEDURE);
 396         }
 397         return refNameServiceTypes;
 398     }
 399
 400     // Seems like a good value - no real data to set this well.
 401     // Note: can set this value lower during debugging; e.g. to 3 - ADR 2012-07-10
 402     private static final int N_OBJS_TO_UPDATE_PER_LOOP = 100;
 403
 404     public static int updateAuthorityRefDocs(
 405             ServiceContext<PoxPayloadIn, PoxPayloadOut> ctx,
 406             RepositoryClient<PoxPayloadIn, PoxPayloadOut> repoClient,
 407             CoreSessionInterface repoSession,
 408             String oldRefName,
 409             String newRefName,
 410             String refPropName) throws Exception {
 411         Map<String, ServiceBindingType> queriedServiceBindings = new HashMap<String, ServiceBindingType>();
 412         Map<String, List<AuthRefConfigInfo>> authRefFieldsByService = new HashMap<String, List<AuthRefConfigInfo>>();
 413
 414         int docsScanned = 0;
 415         int nRefsFound = 0;
 416         int currentPage = 0;
 417         int docsInCurrentPage = 0;
 418         final String WHERE_CLAUSE_ADDITIONS_VALUE = null;
 419         final String ORDER_BY_VALUE = CollectionSpaceClient.CORE_CREATED_AT  // "collectionspace_core:createdAt";
 420                                           + ", " + IQueryManager.NUXEO_UUID; // CSPACE-6333: Add secondary sort on uuid, in case records have the same createdAt timestamp.
 421
 422         if (repoClient instanceof RepositoryJavaClientImpl == false) {
 423             throw new InternalError("updateAuthorityRefDocs() called with unknown repoClient type!");
 424         }
 425
 426         try { // REM - How can we deal with transaction and timeout issues here?
 427             final int pageSize = N_OBJS_TO_UPDATE_PER_LOOP;
 428             DocumentModelList docList;
 429             boolean morePages = true;
 430             while (morePages) {
 431
 432                 docList = findAuthorityRefDocs(ctx, repoClient, repoSession,
 433                         getRefNameServiceTypes(), oldRefName, refPropName,
 434                         queriedServiceBindings, authRefFieldsByService, WHERE_CLAUSE_ADDITIONS_VALUE, ORDER_BY_VALUE, pageSize, currentPage, false);
 435
 436                 if (docList == null) {
 437                     logger.debug("updateAuthorityRefDocs: no documents could be found that referenced the old refName");
 438                     break;
 439                 }
 440                 docsInCurrentPage = docList.size();
 441                 logger.debug("updateAuthorityRefDocs: current page=" + currentPage + " documents included in page=" + docsInCurrentPage);
 442                 if (docsInCurrentPage == 0) {
 443                     logger.debug("updateAuthorityRefDocs: no more documents requiring refName updates could be found");
 444                     break;
 445                 }
 446                 if (docsInCurrentPage < pageSize) {
 447                     logger.debug("updateAuthorityRefDocs: assuming no more documents requiring refName updates will be found, as docsInCurrentPage < pageSize");
 448                     morePages = false;
 449                 }
 450
 451                 // Only match complete refNames - unless and until we decide how to resolve changes
 452                 // to NPTs we will defer that and only change PTs or refNames as passed in.
 453                 int nRefsFoundThisPage = processRefObjsDocListForUpdate(docList, ctx.getTenantId(), oldRefName,
 454                                 queriedServiceBindings, authRefFieldsByService, // Perform the refName updates on the list of document models
 455                         newRefName);
 456                 if (nRefsFoundThisPage > 0) {
 457                     ((RepositoryJavaClientImpl) repoClient).saveDocListWithoutHandlerProcessing(ctx, repoSession, docList, true); // Flush the document model list out to Nuxeo storage
 458                     nRefsFound += nRefsFoundThisPage;
 459                 }
 460
 461                 // FIXME: Per REM, set a limit of num objects - something like
 462                 // 1000K objects - and also add a log Warning after some threshold
 463                 docsScanned += docsInCurrentPage;
 464                 if (morePages) {
 465                     currentPage++;
 466                 }
 467
 468             }
 469         } catch (Exception e) {
 470             logger.error("Internal error updating the AuthorityRefDocs: " + e.getLocalizedMessage());
 471             logger.debug(Tools.errorToString(e, true));
 472             throw e;
 473         }
 474         logger.debug("updateAuthorityRefDocs replaced a total of " + nRefsFound + " authority references, within as many as " + docsScanned + " scanned document(s)");
 475         return nRefsFound;
 476     }
 477
 478     private static DocumentModelList findAllAuthorityRefDocs(
 479             ServiceContext<PoxPayloadIn, PoxPayloadOut> ctx,
 480             RepositoryClient<PoxPayloadIn, PoxPayloadOut> repoClient,
 481             CoreSessionInterface repoSession, List<String> serviceTypes,
 482             String refName,
 483             String refPropName,
 484             Map<String, ServiceBindingType> queriedServiceBindings,
 485             Map<String, List<AuthRefConfigInfo>> authRefFieldsByService,
 486             String whereClauseAdditions,
 487             String orderByClause,
 488             int pageSize,
 489             boolean computeTotal) throws DocumentException, DocumentNotFoundException {
 490
 491         return new LazyAuthorityRefDocList(ctx, repoClient, repoSession,
 492                         serviceTypes, refName, refPropName, queriedServiceBindings, authRefFieldsByService,
 493                         whereClauseAdditions, orderByClause, pageSize, computeTotal);
 494     }
 495
 496     protected static DocumentModelList findAuthorityRefDocs(
 497             ServiceContext<PoxPayloadIn, PoxPayloadOut> ctx,
 498             RepositoryClient<PoxPayloadIn, PoxPayloadOut> repoClient,
 499             CoreSessionInterface repoSession, List<String> serviceTypes,
 500             String refName,
 501             String refPropName,
 502             Map<String, ServiceBindingType> queriedServiceBindings,
 503             Map<String, List<AuthRefConfigInfo>> authRefFieldsByService,
 504             String whereClauseAdditions,
 505             String orderByClause,
 506             int pageSize,
 507             int pageNum,
 508             boolean computeTotal) throws DocumentException, DocumentNotFoundException {
 509
 510         // Get the service bindings for this tenant
 511         TenantBindingConfigReaderImpl tReader =
 512                 ServiceMain.getInstance().getTenantBindingConfigReader();
 513         // We need to get all the procedures, authorities, and objects.
 514         List<ServiceBindingType> servicebindings = tReader.getServiceBindingsByType(ctx.getTenantId(), serviceTypes);
 515         if (servicebindings == null || servicebindings.isEmpty()) {
 516             logger.error("RefNameServiceUtils.getAuthorityRefDocs: No services bindings found, cannot proceed!");
 517             return null;
 518         }
 519         // Filter the list for current user rights
 520         servicebindings = SecurityUtils.getReadableServiceBindingsForCurrentUser(servicebindings);
 521
 522         ArrayList<String> docTypes = new ArrayList<String>();
 523
 524         String query = computeWhereClauseForAuthorityRefDocs(refName, refPropName, docTypes, servicebindings, // REM - Side effect that docTypes array gets set.  Any others?
 525                 queriedServiceBindings, authRefFieldsByService);
 526         if (query == null) { // found no authRef fields - nothing to query
 527             return null;
 528         }
 529         // Additional qualifications, like workflow state
 530         if (Tools.notBlank(whereClauseAdditions)) {
 531             query += " AND " + whereClauseAdditions;
 532         }
 533         // Now we have to issue the search
 534         RepositoryJavaClientImpl nuxeoRepoClient = (RepositoryJavaClientImpl) repoClient;
 535         DocumentWrapper<DocumentModelList> docListWrapper = nuxeoRepoClient.findDocs(ctx, repoSession,
 536                 docTypes, query, orderByClause, pageSize, pageNum, computeTotal);
 537         // Now we gather the info for each document into the list and return
 538         DocumentModelList docList = docListWrapper.getWrappedObject();
 539         return docList;
 540     }
 541     private static final boolean READY_FOR_COMPLEX_QUERY = true;
 542
 543     private static String computeWhereClauseForAuthorityRefDocs(
 544             String refName,
 545             String refPropName,
 546             ArrayList<String> docTypes,
 547             List<ServiceBindingType> servicebindings,
 548             Map<String, ServiceBindingType> queriedServiceBindings,
 549             Map<String, List<AuthRefConfigInfo>> authRefFieldsByService) {
 550
 551         boolean fFirst = true;
 552         List<String> authRefFieldPaths;
 553         for (ServiceBindingType sb : servicebindings) {
 554             // Gets the property names for each part, qualified with the part label (which
 555             // is also the table name, the way that the repository works).
 556             authRefFieldPaths =
 557                     ServiceBindingUtils.getAllPartsPropertyValues(sb,
 558                     refPropName, ServiceBindingUtils.QUALIFIED_PROP_NAMES);
 559             if (authRefFieldPaths.isEmpty()) {
 560                 continue;
 561             }
 562             ArrayList<AuthRefConfigInfo> authRefsInfo = new ArrayList<AuthRefConfigInfo>();
 563             for (String spec : authRefFieldPaths) {
 564                 AuthRefConfigInfo arci = new AuthRefConfigInfo(spec);
 565                 authRefsInfo.add(arci);
 566             }
 567
 568             String docType = sb.getObject().getName();
 569             queriedServiceBindings.put(docType, sb);
 570             authRefFieldsByService.put(docType, authRefsInfo);
 571             docTypes.add(docType);
 572             fFirst = false;
 573         }
 574         if (fFirst) { // found no authRef fields - nothing to query
 575             return null;
 576         }
 577         // We used to build a complete matches query, but that was too complex.
 578         // Just build a keyword query based upon some key pieces - the urn syntax elements and the shortID
 579         // Note that this will also match the Item itself, but that will get filtered out when
 580         // we compute actual matches.
 581         AuthorityTermInfo authTermInfo = RefNameUtils.parseAuthorityTermInfo(refName);
 582
 583         String keywords = RefNameUtils.URN_PREFIX
 584                 + " AND " + (authTermInfo.inAuthority.name != null
 585                 ? authTermInfo.inAuthority.name : authTermInfo.inAuthority.csid)
 586                 + " AND " + (authTermInfo.name != null
 587                 ? authTermInfo.name : authTermInfo.csid);
 588
 589         String whereClauseStr = QueryManager.createWhereClauseFromKeywords(keywords);
 590
 591         if (logger.isTraceEnabled()) {
 592             logger.trace("The 'where' clause to find refObjs is: ", whereClauseStr);
 593         }
 594
 595         return whereClauseStr;
 596     }
 597
 598     // TODO there are multiple copies of this that should be put somewhere common.
 599         protected static String getRefname(DocumentModel docModel) throws ClientException {
 600                 String result = (String)docModel.getProperty(CollectionSpaceClient.COLLECTIONSPACE_CORE_SCHEMA,
 601                                 CollectionSpaceClient.COLLECTIONSPACE_CORE_REFNAME);
 602                 return result;
 603         }
 604
 605     private static int processRefObjsDocListForUpdate(
 606             DocumentModelList docList,
 607             String tenantId,
 608             String refName,
 609             Map<String, ServiceBindingType> queriedServiceBindings,
 610             Map<String, List<AuthRefConfigInfo>> authRefFieldsByService,
 611             String newAuthorityRefName) {
 612         return processRefObjsDocList(docList, tenantId, refName, false, queriedServiceBindings,
 613                         authRefFieldsByService, null, 0, 0, newAuthorityRefName);
 614     }
 615
 616     private static int processRefObjsDocListForList(
 617             DocumentModelList docList,
 618             String tenantId,
 619             String refName,
 620             Map<String, ServiceBindingType> queriedServiceBindings,
 621             Map<String, List<AuthRefConfigInfo>> authRefFieldsByService,
 622             List<AuthorityRefDocList.AuthorityRefDocItem> list,
 623             int pageSize, int pageNum) {
 624         return processRefObjsDocList(docList, tenantId, refName, true, queriedServiceBindings,
 625                         authRefFieldsByService, list, pageSize, pageNum, null);
 626     }
 627
 628
 629         /*
 630      * Runs through the list of found docs, processing them. If list is
 631      * non-null, then processing means gather the info for items. If list is
 632      * null, and newRefName is non-null, then processing means replacing and
 633      * updating. If processing/updating, this must be called in the context of
 634      * an open session, and caller must release Session after calling this.
 635      *
 636      */
 637     private static int processRefObjsDocList(
 638             DocumentModelList docList,
 639             String tenantId,
 640             String refName,
 641             boolean matchBaseOnly,
 642             Map<String, ServiceBindingType> queriedServiceBindings,
 643             Map<String, List<AuthRefConfigInfo>> authRefFieldsByService,
 644             List<AuthorityRefDocList.AuthorityRefDocItem> list,
 645             int pageSize, int pageNum,  // Only used when constructing a list.
 646             String newAuthorityRefName) {
 647         UriTemplateRegistry registry = ServiceMain.getInstance().getUriTemplateRegistry();
 648         Iterator<DocumentModel> iter = docList.iterator();
 649         int nRefsFoundTotal = 0;
 650         boolean foundSelf = false;
 651
 652         // When paginating results, we have to guess at the total. First guess is the number of docs returned
 653         // by the query. However, this returns some false positives, so may be high.
 654         // In addition, we can match multiple fields per doc, so this may be low. Fun, eh?
 655         int nDocsReturnedInQuery = (int)docList.totalSize();
 656         int nDocsProcessed = 0;
 657         int firstItemInPage = pageNum*pageSize;
 658         while (iter.hasNext()) {
 659             DocumentModel docModel = iter.next();
 660             AuthorityRefDocList.AuthorityRefDocItem ilistItem;
 661
 662             String docType = docModel.getDocumentType().getName(); // REM - This will be a tentant qualified document type
 663             docType = ServiceBindingUtils.getUnqualifiedTenantDocType(docType);
 664             ServiceBindingType sb = queriedServiceBindings.get(docType);
 665             if (sb == null) {
 666                 throw new RuntimeException(
 667                         "getAuthorityRefDocs: No Service Binding for docType: " + docType);
 668             }
 669
 670             if (list == null) { // no list - should be update refName case.
 671                 if (newAuthorityRefName == null) {
 672                     throw new InternalError("processRefObjsDocList() called with neither an itemList nor a new RefName!");
 673                 }
 674                 ilistItem = null;
 675                 pageSize = 0;
 676                 firstItemInPage = 0;    // Do not paginate if updating, rather than building list
 677             } else {    // Have a list - refObjs case
 678                 if (newAuthorityRefName != null) {
 679                     throw new InternalError("processRefObjsDocList() called with both an itemList and a new RefName!");
 680                 }
 681                 if(firstItemInPage > 100) {
 682                         logger.warn("Processing a large offset (size:{}, num:{}) for refObjs - will be expensive!!!",
 683                                                 pageSize, pageNum);
 684                 }
 685                 // Note that we have to go through check all the fields to determine the actual page start
 686                 ilistItem = new AuthorityRefDocList.AuthorityRefDocItem();
 687                 String csid = NuxeoUtils.getCsid(docModel);//NuxeoUtils.extractId(docModel.getPathAsString());
 688                 try {
 689                         String itemRefName = getRefname(docModel);
 690                         ilistItem.setRefName(itemRefName);
 691                 } catch (ClientException ce) {
 692                     throw new RuntimeException(
 693                             "processRefObjsDocList: Problem fetching refName from item Object: "
 694                                         + ce.getLocalizedMessage());
 695                 }
 696                 ilistItem.setDocId(csid);
 697                 String uri = "";
 698                 UriTemplateRegistryKey key = new UriTemplateRegistryKey(tenantId, docType);
 699                 StoredValuesUriTemplate template = registry.get(key);
 700                 if (template != null) {
 701                     Map<String, String> additionalValues = new HashMap<String, String>();
 702                     if (template.getUriTemplateType() == UriTemplateFactory.RESOURCE) {
 703                         additionalValues.put(UriTemplateFactory.IDENTIFIER_VAR, csid);
 704                         uri = template.buildUri(additionalValues);
 705                     } else if (template.getUriTemplateType() == UriTemplateFactory.ITEM) {
 706                         try {
 707                             String inAuthorityCsid = (String) NuxeoUtils.getProperyValue(docModel, "inAuthority"); //docModel.getPropertyValue("inAuthority"); // AuthorityItemJAXBSchema.IN_AUTHORITY
 708                             additionalValues.put(UriTemplateFactory.IDENTIFIER_VAR, inAuthorityCsid);
 709                             additionalValues.put(UriTemplateFactory.ITEM_IDENTIFIER_VAR, csid);
 710                             uri = template.buildUri(additionalValues);
 711                         } catch (Exception e) {
 712                             logger.warn("Could not extract inAuthority property from authority item record: " + e.getMessage());
 713                         }
 714                     } else if (template.getUriTemplateType() == UriTemplateFactory.CONTACT) {
 715                         // FIXME: Generating contact sub-resource URIs requires additional work,
 716                         // as a follow-on to CSPACE-5271 - ADR 2012-08-16
 717                         // Sets the default (empty string) value for uri, for now
 718                     } else {
 719                         logger.warn("Unrecognized URI template type = " + template.getUriTemplateType());
 720                         // Sets the default (empty string) value for uri
 721                     }
 722                 } else { // (if template == null)
 723                     logger.warn("Could not retrieve URI template from registry via tenant ID "
 724                             + tenantId + " and docType " + docType);
 725                     // Sets the default (empty string) value for uri
 726                 }
 727                 ilistItem.setUri(uri);
 728                 try {
 729                     ilistItem.setWorkflowState(docModel.getCurrentLifeCycleState());
 730                     ilistItem.setUpdatedAt(NuxeoDocumentModelHandler.getUpdatedAtAsString(docModel));
 731                 } catch (Exception e) {
 732                     logger.error("Error getting core values for doc [" + csid + "]: " + e.getLocalizedMessage());
 733                 }
 734                 ilistItem.setDocType(docType);
 735                 ilistItem.setDocNumber(
 736                         ServiceBindingUtils.getMappedFieldInDoc(sb, ServiceBindingUtils.OBJ_NUMBER_PROP, docModel));
 737                 ilistItem.setDocName(
 738                         ServiceBindingUtils.getMappedFieldInDoc(sb, ServiceBindingUtils.OBJ_NAME_PROP, docModel));
 739             }
 740             // Now, we have to loop over the authRefFieldsByService to figure
 741             // out which field(s) matched this.
 742             List<AuthRefConfigInfo> matchingAuthRefFields = authRefFieldsByService.get(docType);
 743             if (matchingAuthRefFields == null || matchingAuthRefFields.isEmpty()) {
 744                 throw new RuntimeException(
 745                         "getAuthorityRefDocs: internal logic error: can't fetch authRefFields for DocType.");
 746             }
 747             //String authRefAncestorField = "";
 748             //String authRefDescendantField = "";
 749             //String sourceField = "";
 750
 751             ArrayList<RefNameServiceUtils.AuthRefInfo> foundProps = new ArrayList<RefNameServiceUtils.AuthRefInfo>();
 752             try {
 753                 findAuthRefPropertiesInDoc(docModel, matchingAuthRefFields, refName, matchBaseOnly, foundProps); // REM - side effect that foundProps is set
 754                 if(!foundProps.isEmpty()) {
 755                     int nRefsFoundInDoc = 0;
 756                         for (RefNameServiceUtils.AuthRefInfo ari : foundProps) {
 757                                 if (ilistItem != null) {
 758                                         // So this is a true positive, and not a false one. We have to consider pagination now.
 759                                         if(nRefsFoundTotal >= firstItemInPage) {        // skipped enough already
 760                                                 if (nRefsFoundInDoc == 0) {    // First one?
 761                                                         ilistItem.setSourceField(ari.getQualifiedDisplayName());
 762                                                 } else {    // duplicates from one object
 763                                                         ilistItem = cloneAuthRefDocItem(ilistItem, ari.getQualifiedDisplayName());
 764                                                 }
 765                                                 list.add(ilistItem);
 766                                         nRefsFoundInDoc++;      // Only increment if processed, or clone logic above will fail
 767                                         }
 768                                 } else {    // update refName case
 769                                         Property propToUpdate = ari.getProperty();
 770                                         propToUpdate.setValue(newAuthorityRefName);
 771                                 }
 772                                 nRefsFoundTotal++;              // Whether we processed or not, we found - essential to pagination logic
 773                         }
 774                 } else if(ilistItem != null) {
 775                         String docRefName = ilistItem.getRefName();
 776                     if (matchBaseOnly?
 777                                         (docRefName!=null && docRefName.startsWith(refName))
 778                                         :refName.equals(docRefName)) {
 779                                 // We found the self for an item
 780                                 foundSelf = true;
 781                                 logger.debug("getAuthorityRefDocs: Result: "
 782                                                                 + docType + " [" + NuxeoUtils.getCsid(docModel)
 783                                                                 + "] appears to be self for: ["
 784                                                                 + refName + "]");
 785                         } else {
 786                                 logger.debug("getAuthorityRefDocs: Result: "
 787                                                                 + docType + " [" + NuxeoUtils.getCsid(docModel)
 788                                                                 + "] does not reference ["
 789                                                                 + refName + "]");
 790                         }
 791                 }
 792             } catch (ClientException ce) {
 793                 throw new RuntimeException(
 794                                 "getAuthorityRefDocs: Problem fetching values from repo: " + ce.getLocalizedMessage());
 795             }
 796             nDocsProcessed++;
 797             // Done processing that doc. Are we done with the whole page?
 798             // Note pageSize <=0 means do them all
 799             if((pageSize > 0) && ((nRefsFoundTotal-firstItemInPage)>=pageSize)) {
 800                 // Quitting early, so we need to estimate the total. Assume one per doc
 801                 // for the rest of the docs we matched in the query
 802                 int unprocessedDocs = nDocsReturnedInQuery - nDocsProcessed;
 803                 if(unprocessedDocs>0) {
 804                         // We generally match ourselves in the keyword search. If we already saw ourselves
 805                         // then do not try to correct for this. Otherwise, decrement the total.
 806                         // Yes, this is fairly goofy, but the whole estimation mechanism is goofy.
 807                         if(!foundSelf)
 808                                 unprocessedDocs--;
 809                         nRefsFoundTotal += unprocessedDocs;
 810                 }
 811                 break;
 812             }
 813         } // close while(iterator)
 814         return nRefsFoundTotal;
 815     }
 816
 817     private static AuthorityRefDocList.AuthorityRefDocItem cloneAuthRefDocItem(
 818             AuthorityRefDocList.AuthorityRefDocItem ilistItem, String sourceField) {
 819         AuthorityRefDocList.AuthorityRefDocItem newlistItem = new AuthorityRefDocList.AuthorityRefDocItem();
 820         newlistItem.setDocId(ilistItem.getDocId());
 821         newlistItem.setDocName(ilistItem.getDocName());
 822         newlistItem.setDocNumber(ilistItem.getDocNumber());
 823         newlistItem.setDocType(ilistItem.getDocType());
 824         newlistItem.setUri(ilistItem.getUri());
 825         newlistItem.setSourceField(sourceField);
 826         return newlistItem;
 827     }
 828
 829     public static List<AuthRefInfo> findAuthRefPropertiesInDoc(
 830             DocumentModel docModel,
 831             List<AuthRefConfigInfo> authRefFieldInfo,
 832             String refNameToMatch,
 833             List<AuthRefInfo> foundProps) {
 834         return findAuthRefPropertiesInDoc(docModel, authRefFieldInfo,
 835                                                                         refNameToMatch, false, foundProps);
 836     }
 837
 838     public static List<AuthRefInfo> findAuthRefPropertiesInDoc(
 839             DocumentModel docModel,
 840             List<AuthRefConfigInfo> authRefFieldInfo,
 841             String refNameToMatch,
 842             boolean matchBaseOnly,
 843             List<AuthRefInfo> foundProps) {
 844         // Assume that authRefFieldInfo is keyed by the field name (possibly mapped for UI)
 845         // and the values are elPaths to the field, where intervening group structures in
 846         // lists of complex structures are replaced with "*". Thus, valid paths include
 847         // the following (note that the ServiceBindingUtils prepend schema names to configured values):
 848         // "schemaname:fieldname"
 849         // "schemaname:scalarlistname"
 850         // "schemaname:complexfieldname/fieldname"
 851         // "schemaname:complexlistname/*/fieldname"
 852         // "schemaname:complexlistname/*/scalarlistname"
 853         // "schemaname:complexlistname/*/complexfieldname/fieldname"
 854         // "schemaname:complexlistname/*/complexlistname/*/fieldname"
 855         // etc.
 856         for (AuthRefConfigInfo arci : authRefFieldInfo) {
 857             try {
 858                 // Get first property and work down as needed.
 859                 Property prop = docModel.getProperty(arci.pathEls[0]);
 860                 findAuthRefPropertiesInProperty(foundProps, prop, arci, 0, refNameToMatch, matchBaseOnly);
 861             } catch (Exception e) {
 862                 logger.error("Problem fetching property: " + arci.pathEls[0]);
 863             }
 864         }
 865         return foundProps;
 866     }
 867
 868     private static List<AuthRefInfo> findAuthRefPropertiesInProperty(
 869             List<AuthRefInfo> foundProps,
 870             Property prop,
 871             AuthRefConfigInfo arci,
 872             int pathStartIndex, // Supports recursion and we work down the path
 873             String refNameToMatch,
 874             boolean matchBaseOnly ) {
 875         if (pathStartIndex >= arci.pathEls.length) {
 876             throw new ArrayIndexOutOfBoundsException("Index = " + pathStartIndex + " for path: "
 877                     + arci.pathEls.toString());
 878         }
 879         AuthRefInfo ari = null;
 880         if (prop == null) {
 881             return foundProps;
 882         }
 883
 884         if (prop instanceof StringProperty) {    // scalar string
 885             addARIifMatches(refNameToMatch, matchBaseOnly, arci, prop, foundProps);
 886         } else if (prop instanceof List) {
 887             List<Property> propList = (List<Property>) prop;
 888             // run through list. Must either be list of Strings, or Complex
 889             for (Property listItemProp : propList) {
 890                 if (listItemProp instanceof StringProperty) {
 891                     if (arci.pathEls.length - pathStartIndex != 1) {
 892                         logger.error("Configuration for authRefs does not match schema structure: "
 893                                 + arci.pathEls.toString());
 894                         break;
 895                     } else {
 896                         addARIifMatches(refNameToMatch, matchBaseOnly, arci, listItemProp, foundProps);
 897                     }
 898                 } else if (listItemProp.isComplex()) {
 899                     // Just recurse to handle this. Note that since this is a list of complex,
 900                     // which should look like listName/*/... we add 2 to the path start index
 901                     findAuthRefPropertiesInProperty(foundProps, listItemProp, arci,
 902                             pathStartIndex + 2, refNameToMatch, matchBaseOnly);
 903                 } else {
 904                     logger.error("Configuration for authRefs does not match schema structure: "
 905                             + arci.pathEls.toString());
 906                     break;
 907                 }
 908             }
 909         } else if (prop.isComplex()) {
 910             String localPropName = arci.pathEls[pathStartIndex];
 911             try {
 912                 Property localProp = prop.get(localPropName);
 913                 // Now just recurse, pushing down the path 1 step
 914                 findAuthRefPropertiesInProperty(foundProps, localProp, arci,
 915                         pathStartIndex, refNameToMatch, matchBaseOnly);
 916             } catch (PropertyNotFoundException pnfe) {
 917                 logger.error("Could not find property: [" + localPropName + "] in path: "
 918                         + arci.getFullPath());
 919                 // Fall through - ari will be null and we will continue...
 920             }
 921         } else {
 922             logger.error("Configuration for authRefs does not match schema structure: "
 923                     + arci.pathEls.toString());
 924         }
 925
 926         if (ari != null) {
 927             foundProps.add(ari); //FIXME: REM - This is dead code.  'ari' is never touched after being initalized to null.  Why?
 928         }
 929
 930         return foundProps;
 931     }
 932
 933     private static void addARIifMatches(
 934             String refNameToMatch,
 935             boolean matchBaseOnly,
 936             AuthRefConfigInfo arci,
 937             Property prop,
 938             List<AuthRefInfo> foundProps) {
 939         // Need to either match a passed refName
 940         // OR have no refName to match but be non-empty
 941         try {
 942             String value = (String) prop.getValue();
 943             if (((refNameToMatch != null) &&
 944                                 (matchBaseOnly?
 945                                         (value!=null && value.startsWith(refNameToMatch))
 946                                         :refNameToMatch.equals(value)))
 947                     || ((refNameToMatch == null) && Tools.notBlank(value))) {
 948                 // Found a match
 949                 logger.debug("Found a match on property: " + prop.getPath() + " with value: [" + value + "]");
 950                 AuthRefInfo ari = new AuthRefInfo(arci, prop);
 951                 foundProps.add(ari);
 952             }
 953         } catch (PropertyException pe) {
 954             logger.debug("PropertyException on: " + prop.getPath() + pe.getLocalizedMessage());
 955         }
 956     }
 957
 958     /*
 959      * Identifies whether the refName was found in the supplied field. If passed
 960      * a new RefName, will set that into fields in which the old one was found.
 961      *
 962      * Only works for: * Scalar fields * Repeatable scalar fields (aka
 963      * multi-valued fields)
 964      *
 965      * Does not work for: * Structured fields (complexTypes) * Repeatable
 966      * structured fields (repeatable complexTypes) private static int
 967      * refNameFoundInField(String oldRefName, Property fieldValue, String
 968      * newRefName) { int nFound = 0; if (fieldValue instanceof List) {
 969      * List<Property> fieldValueList = (List) fieldValue; for (Property
 970      * listItemValue : fieldValueList) { try { if ((listItemValue instanceof
 971      * StringProperty) &&
 972      * oldRefName.equalsIgnoreCase((String)listItemValue.getValue())) {
 973      * nFound++; if(newRefName!=null) { fieldValue.setValue(newRefName); } else
 974      * { // We cannot quit after the first, if we are replacing values. // If we
 975      * are just looking (not replacing), finding one is enough. break; } } }
 976      * catch( PropertyException pe ) {} } } else { try { if ((fieldValue
 977      * instanceof StringProperty) &&
 978      * oldRefName.equalsIgnoreCase((String)fieldValue.getValue())) { nFound++;
 979      * if(newRefName!=null) { fieldValue.setValue(newRefName); } } } catch(
 980      * PropertyException pe ) {} } return nFound; }
 981      */
 982 }