From fa4bdc7eb5e66a718495876efe2c665f8fd8b5ac Mon Sep 17 00:00:00 2001 From: remillet Date: Mon, 16 Oct 2017 17:09:54 -0700 Subject: [PATCH] NOJIRA: Reduced default output of Import service. Added support for impTimeout and impTimout query params. Fixed error handling of Import service POST requests. --- .../services/client/IClientQueryParams.java | 6 +- .../services/common/api/FileTools.java | 9 +- .../context/AbstractServiceContextImpl.java | 9 +- .../services/imports/ImportsResource.java | 19 ++-- .../services/imports/nuxeo/ImportCommand.java | 34 +------ .../imports/nuxeo/LoggedXMLZipReader.java | 95 +++++++++++++++++++ 6 files changed, 129 insertions(+), 43 deletions(-) create mode 100644 services/imports/service/src/main/java/org/collectionspace/services/imports/nuxeo/LoggedXMLZipReader.java diff --git a/services/client/src/main/java/org/collectionspace/services/client/IClientQueryParams.java b/services/client/src/main/java/org/collectionspace/services/client/IClientQueryParams.java index 52c4ee623..520e80c80 100644 --- a/services/client/src/main/java/org/collectionspace/services/client/IClientQueryParams.java +++ b/services/client/src/main/java/org/collectionspace/services/client/IClientQueryParams.java @@ -31,7 +31,11 @@ public interface IClientQueryParams { public static final String PAGE_SIZE_PARAM = "pgSz"; public static final String START_PAGE_PARAM = "pgNum"; public static final String ORDER_BY_PARAM = "sortBy"; - public static final String IMPORT_TIMEOUT_PARAM = "impTimout"; + + @Deprecated + public static final String IMPORT_TIMOUT_PARAM = "impTimout"; + public static final String IMPORT_TIMEOUT_PARAM = "impTimeout"; + public static final String UPDATE_CORE_VALUES = "updateCoreValues"; public static final String FORCE_REFNAME_UPDATES = "forceRefnameUpdates"; public static final String FORCE_SYCN = "forceSync"; diff --git a/services/common-api/src/main/java/org/collectionspace/services/common/api/FileTools.java b/services/common-api/src/main/java/org/collectionspace/services/common/api/FileTools.java index cc468f85a..95fa0a83b 100644 --- a/services/common-api/src/main/java/org/collectionspace/services/common/api/FileTools.java +++ b/services/common-api/src/main/java/org/collectionspace/services/common/api/FileTools.java @@ -41,13 +41,16 @@ import java.util.UUID; import javax.xml.bind.JAXBContext; import javax.xml.bind.Unmarshaller; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * @author Laramie Crocker * $LastChangedRevision: $ * $LastChangedDate: $ */ public class FileTools { - + private static final Logger logger = LoggerFactory.getLogger(FileTools.class); public static String DEFAULT_ENCODING = ""; public static String UTF8_ENCODING = "UTF-8"; public static Charset UTF8_CHARSET = java.nio.charset.StandardCharsets.UTF_8; @@ -122,7 +125,9 @@ public class FileTools { if (parent != null){ File p = new File(parent); p.mkdirs(); - System.out.println("Making directory: "+p.getCanonicalPath()); + if (logger.isTraceEnabled()) { + logger.trace("Making directory: " + p.getCanonicalPath()); + } } } diff --git a/services/common/src/main/java/org/collectionspace/services/common/context/AbstractServiceContextImpl.java b/services/common/src/main/java/org/collectionspace/services/common/context/AbstractServiceContextImpl.java index 75af571d7..24e37fbf6 100644 --- a/services/common/src/main/java/org/collectionspace/services/common/context/AbstractServiceContextImpl.java +++ b/services/common/src/main/java/org/collectionspace/services/common/context/AbstractServiceContextImpl.java @@ -186,13 +186,18 @@ public abstract class AbstractServiceContextImpl MultivaluedMap queryParams = (ui == null) ? null : ui.getQueryParameters(); if (queryParams != null) { String timeoutString = queryParams.getFirst(IClientQueryParams.IMPORT_TIMEOUT_PARAM); - if (timeoutString != null) - try { + if (timeoutString == null) { + timeoutString = queryParams.getFirst(IClientQueryParams.IMPORT_TIMOUT_PARAM); + } + + if (timeoutString != null) { + try { result = Integer.parseInt(timeoutString); } catch (NumberFormatException e) { logger.warn("Transaction timeout period parameter could not be parsed. The characters in the parameter string must all be decimal digits. The Import service will use the default timeout period instead.", e); } + } } return result; diff --git a/services/imports/service/src/main/java/org/collectionspace/services/imports/ImportsResource.java b/services/imports/service/src/main/java/org/collectionspace/services/imports/ImportsResource.java index 3253967e0..adc5e8ed4 100644 --- a/services/imports/service/src/main/java/org/collectionspace/services/imports/ImportsResource.java +++ b/services/imports/service/src/main/java/org/collectionspace/services/imports/ImportsResource.java @@ -203,22 +203,23 @@ public class ImportsResource extends AbstractCollectionSpaceResourceImpl ctx = createServiceContext(ui); + ServiceContext ctx = createServiceContext(ui); int timeout = ctx.getTimeoutSecs(); // gets it from query param 'impTimout' or uses default if no query param specified + // InputSource inputSource = payloadToInputSource(xmlPayload); // result = createFromInputSource(inputSource); String inputFilename = payloadToFilename(xmlPayload); result = createFromFilename(inputFilename, timeout); + rb.entity(result); } catch (Exception e) { - result = e.getMessage(); - logger.error(result); - rb = Response.status(Response.Status.INTERNAL_SERVER_ERROR); - } finally { - rb.entity(result); - return rb.build(); - } + result = e.getMessage(); + logger.error(result); + } + + return rb.build(); } public static String createFromInputSource(InputSource inputSource, diff --git a/services/imports/service/src/main/java/org/collectionspace/services/imports/nuxeo/ImportCommand.java b/services/imports/service/src/main/java/org/collectionspace/services/imports/nuxeo/ImportCommand.java index b22e5cf09..4aa5fcb9e 100644 --- a/services/imports/service/src/main/java/org/collectionspace/services/imports/nuxeo/ImportCommand.java +++ b/services/imports/service/src/main/java/org/collectionspace/services/imports/nuxeo/ImportCommand.java @@ -1,7 +1,6 @@ package org.collectionspace.services.imports.nuxeo; import java.io.File; -import java.io.IOException; import java.util.Calendar; import java.util.HashMap; import java.util.Map; @@ -10,21 +9,14 @@ import java.util.TreeSet; import org.collectionspace.services.nuxeo.client.java.NuxeoClientEmbedded; import org.collectionspace.services.nuxeo.client.java.NuxeoConnectorEmbedded; import org.collectionspace.services.nuxeo.client.java.CoreSessionInterface; -import org.nuxeo.common.utils.FileUtils; import org.nuxeo.ecm.core.api.DocumentModel; import org.nuxeo.ecm.core.api.DocumentRef; -import org.nuxeo.ecm.core.api.IdRef; -import org.nuxeo.ecm.core.api.PathRef; import org.nuxeo.ecm.core.io.DocumentPipe; import org.nuxeo.ecm.core.io.DocumentReader; import org.nuxeo.ecm.core.io.DocumentTranslationMap; import org.nuxeo.ecm.core.io.DocumentWriter; import org.nuxeo.ecm.core.io.impl.DocumentPipeImpl; import org.nuxeo.ecm.core.io.impl.plugins.DocumentModelWriter; -import org.nuxeo.ecm.platform.importer.service.DefaultImporterService; -import org.nuxeo.ecm.platform.importer.service.DefaultImporterServiceImpl; -import org.nuxeo.ecm.platform.importer.xml.parser.XMLImporterService; -import org.nuxeo.runtime.api.Framework; // we use our own override of this: import org.nuxeo.ecm.core.io.impl.plugins.XMLDirectoryReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,7 +37,7 @@ public class ImportCommand { String msg = String.format("Start of import is Local time: %tT", Calendar.getInstance()); logger.debug(msg); } - return newImportTree(repoSession, file, workspacesPath, timeout); + return importTree(repoSession, file, workspacesPath, timeout); } catch (Exception e) { throw e; } finally { @@ -56,26 +48,6 @@ public class ImportCommand { client.releaseRepository(repoSession); } } - -// DocumentModel root = session.getRootDocument(); -// File xml = File("/where/is/my/umbrella"); -// XMLImporterService importer = Framework.getLocalService(XMLImporterService.class); -// importer.importDocuments(root, xml); - - private String newImportTree(CoreSessionInterface repoSession, File file, String toPath, int timeout) throws IOException { - File source = new File("C:/dev/tools/apache-tomcat-7.0.64/temp/imports-2014210770025398386/Personauthorities.zip"); - if (source.exists()) { - String targetPath = "/core-domain/Workspaces/Personauthorities/"; ///core-domain/Workspaces/Personauthorities - DocumentRef docRef = new PathRef(targetPath); - DocumentModel root = repoSession.getDocument(docRef); - XMLImporterService importerService = Framework.getLocalService(XMLImporterService.class); - importerService.importDocuments(root, source); - } else { - System.err.println("Could not find file: " + source.getAbsolutePath()); - } - - return "New Import Service Run"; - } /* * If the import exceeds the number of seconds in 'timeout', we'll thrown an exception and rollback all import work @@ -102,7 +74,11 @@ public class ImportCommand { // pipe.addTransformer(transformer); pipe.setReader(reader); pipe.setWriter(writer); + + logger.trace(String.format(">>> Pipe importer start: %d", System.currentTimeMillis())); DocumentTranslationMap dtm = pipe.run(); + logger.trace(String.format("<<< Pipe importer stop: %d", System.currentTimeMillis())); + if (dtm == null) { throw new Exception("Could not process import payload. Check XML markup for not-well-formed errors, elements not matching import schema, etc."); } diff --git a/services/imports/service/src/main/java/org/collectionspace/services/imports/nuxeo/LoggedXMLZipReader.java b/services/imports/service/src/main/java/org/collectionspace/services/imports/nuxeo/LoggedXMLZipReader.java new file mode 100644 index 000000000..641244536 --- /dev/null +++ b/services/imports/service/src/main/java/org/collectionspace/services/imports/nuxeo/LoggedXMLZipReader.java @@ -0,0 +1,95 @@ +package org.collectionspace.services.imports.nuxeo; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +import org.dom4j.Document; +import org.dom4j.DocumentException; +import org.dom4j.io.SAXReader; +import org.nuxeo.common.utils.Path; +import org.nuxeo.ecm.core.io.ExportConstants; +import org.nuxeo.ecm.core.io.ExportedDocument; +import org.nuxeo.ecm.core.io.impl.ExportedDocumentImpl; +import org.nuxeo.ecm.core.io.impl.plugins.XMLZipReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LoggedXMLZipReader extends XMLZipReader { + private final Logger logger = LoggerFactory.getLogger(LoggedXMLZipReader.class); + + private ZipFile zip; + Enumeration entries = null; + + + private LoggedXMLZipReader(ZipFile zip) { + super((ZipFile)null); + this.zip = zip; + } + + public LoggedXMLZipReader(File source) throws IOException { + this(new ZipFile(source)); + entries = zip.entries(); + } + + private List reportList = new ArrayList(); + public String report(){ + StringBuffer result = new StringBuffer(); + for (String s: reportList){ + result.append(s).append("\r\n"); + } + return result.toString(); + } + + @Override + // the zip entry order is the same as one used when creating the zip + public ExportedDocument read() throws IOException { + while (entries.hasMoreElements()) { + ZipEntry entry = entries.nextElement(); + if (entry.isDirectory()) { + return createDocument(entry); + } + } + return null; + } + + private ExportedDocument createDocument(ZipEntry dirEntry) throws IOException { + ExportedDocument xdoc = new ExportedDocumentImpl(); + String dirPath = dirEntry.getName(); + // read the main document + ZipEntry entry = zip.getEntry(dirPath + ExportConstants.DOCUMENT_FILE); + if (entry != null) { + InputStream in = zip.getInputStream(entry); + try { + Document doc = readXML(in); + xdoc.setDocument(doc); + Path relPath = new Path(dirPath).removeTrailingSeparator(); +// xdoc.setPath(new Path(dirPath + relPath).removeTrailingSeparator()); + xdoc.setPath(relPath); + } finally { + in.close(); + } + } + + return xdoc; + } + + @Override + public Document readXML(InputStream in) { + Document result = null; + + try { + result = new SAXReader().read(in); + reportList.add("READ: " + result.getName()); + } catch (DocumentException e) { + reportList.add("ERROR: " + e.getMessage()); + } + + return result; + } +} -- 2.47.3