From 4f164e6a3f9871d8c7f4ff0a7abbcd0a4ff84bd8 Mon Sep 17 00:00:00 2001 From: Richard Millet Date: Thu, 5 May 2011 08:46:01 +0000 Subject: [PATCH] CSPACE-3698: Post to Blob service with URL query param to external image now supported. --- .../services/client/test/BlobServiceTest.java | 21 +- .../services/common/Download.java | 274 ++++++++++++++++++ .../services/common/FileUtils.java | 33 ++- .../services/common/blob/BlobInput.java | 42 ++- .../jaxb/src/main/resources/blobs_common.xsd | 1 + .../client/test/MediaServiceTest.java | 4 +- 6 files changed, 357 insertions(+), 18 deletions(-) create mode 100644 services/common/src/main/java/org/collectionspace/services/common/Download.java diff --git a/services/blob/client/src/test/java/org/collectionspace/services/client/test/BlobServiceTest.java b/services/blob/client/src/test/java/org/collectionspace/services/client/test/BlobServiceTest.java index b9c732cb0..421daa058 100644 --- a/services/blob/client/src/test/java/org/collectionspace/services/client/test/BlobServiceTest.java +++ b/services/blob/client/src/test/java/org/collectionspace/services/client/test/BlobServiceTest.java @@ -23,6 +23,7 @@ package org.collectionspace.services.client.test; import java.io.File; +import java.net.URL; import java.util.List; import javax.ws.rs.core.MediaType; @@ -125,7 +126,7 @@ public class BlobServiceTest extends AbstractServiceTestImpl { * @param fromUri - if 'false' then send the service a multipart/form-data POST from which to create the blob. * @throws Exception the exception */ - protected void createBlob(String testName, boolean fromUri) throws Exception { + protected void createBlob(String testName, boolean fromUri, String uri) throws Exception { setupCreate(); BlobClient client = new BlobClient(); @@ -142,7 +143,13 @@ public class BlobServiceTest extends AbstractServiceTestImpl { logger.debug("Processing file URI: " + child.getAbsolutePath()); logger.debug("MIME type is: " + mimeType); if (fromUri == true) { - res = client.createBlobFromURI(child.getAbsolutePath()); + if (uri != null) { + res = client.createBlobFromURI(uri); + break; + } else { + URL childUrl = child.toURI().toURL(); + res = client.createBlobFromURI(childUrl.toString()); + } } else { MultipartFormDataOutput form = new MultipartFormDataOutput(); OutputPart outputPart = form.addFormData("file", child, MediaType.valueOf(mimeType)); @@ -165,14 +172,20 @@ public class BlobServiceTest extends AbstractServiceTestImpl { @Test(dataProvider = "testName", dataProviderClass = AbstractServiceTestImpl.class, dependsOnMethods = {"create"}) public void createBlobWithURI(String testName) throws Exception { logger.debug(testBanner(testName, CLASS_NAME)); - createBlob(testName, true /*with URI*/); + createBlob(testName, true /*with URI*/, null); } + @Test(dataProvider = "testName", dataProviderClass = AbstractServiceTestImpl.class, dependsOnMethods = {"create"}) + public void createBlobWithURL(String testName) throws Exception { + logger.debug(testBanner(testName, CLASS_NAME)); + createBlob(testName, true /*with URI*/, "http://farm6.static.flickr.com/5289/5688023100_15e00cde47_o.jpg"); + } + @Test(dataProvider = "testName", dataProviderClass = AbstractServiceTestImpl.class, dependsOnMethods = {"createBlobWithURI"}) public void createBlobWithPost(String testName) throws Exception { logger.debug(testBanner(testName, CLASS_NAME)); - createBlob(testName, false /*with POST*/); + createBlob(testName, false /*with POST*/, null); } @Override diff --git a/services/common/src/main/java/org/collectionspace/services/common/Download.java b/services/common/src/main/java/org/collectionspace/services/common/Download.java new file mode 100644 index 000000000..ef6fd8136 --- /dev/null +++ b/services/common/src/main/java/org/collectionspace/services/common/Download.java @@ -0,0 +1,274 @@ +package org.collectionspace.services.common; +import java.io.*; +import java.net.*; +import java.util.*; + +// This class downloads a file from a URL. +public class Download extends Observable implements Runnable { + + public static URL verifyUrl(String url) { + // Only allow HTTP URLs. + if (!url.toLowerCase().startsWith("http://")) + return null; + + // Verify format of URL. + URL verifiedUrl = null; + try { + verifiedUrl = new URL(url); + } catch (Exception e) { + return null; + } + + // Make sure URL specifies a file. + if (verifiedUrl.getFile().length() < 2) + return null; + + return verifiedUrl; + } + + // Max size of download buffer. + private static final int MAX_BUFFER_SIZE = 1024; + + // These are the status names. + public static final String STATUSES[] = {"Downloading", + "Paused", "Complete", "Cancelled", "Error"}; + + // These are the status codes. + public static final int DOWNLOADING = 0; + public static final int PAUSED = 1; + public static final int COMPLETE = 2; + public static final int CANCELLED = 3; + public static final int ERROR = 4; + + private String errorMessage = null; + private String filePath = null; + private File downloadedFile = null; + private String destDir = null; // the location of the downloaded file + private URL url; // download URL + private int size; // size of download in bytes + private int downloaded; // number of bytes downloaded + private int status; // current status of download + + public Download(URL url, String destDir) { + size = -1; + downloaded = 0; + status = DOWNLOADING; + this.url = url; + setDestDir(destDir); + + // Begin the download. + download(); + } + + // Constructor for Download. + public Download(URL url) { + this(url, System.getProperty("java.io.tmpdir") + UUID.randomUUID() + File.separator); + } + + private void setDestDir(String destDir) { + if (destDir.endsWith(File.separator) != true) { + destDir = destDir + File.separator; + } + // + // Check if the destDir exists, if not try to create it. + // + File destDirFile = new File(destDir); + boolean destDirExists = destDirFile.exists(); + if (destDirExists == false) { + destDirExists = destDirFile.mkdir(); + } else { + destDirExists = destDirFile.isDirectory(); + } + // + // If we couldn't create the directory or if it is already a file then fail. + // + if (destDirExists = true) { + this.destDir = destDir; + } else { + error("Could not download file to: " + destDir); + } + } + + + public File getFile() { + return this.downloadedFile; + } + + public String getFilePath() { + return this.filePath; + } + + public String getDestDir() { + return this.destDir; + } + + // Get this download's URL. + public String getUrl() { + return url.toString(); + } + + // Get this download's size. + public int getSize() { + return size; + } + + // Get this download's progress. + public float getProgress() { + return ((float) downloaded / size) * 100; + } + + // Get this download's status. + public int getStatus() { + return status; + } + + // Pause this download. + public void pause() { + status = PAUSED; + stateChanged(); + } + + // Resume this download. + public void resume() { + status = DOWNLOADING; + stateChanged(); + download(); + } + + // Cancel this download. + public void cancel() { + status = CANCELLED; + stateChanged(); + } + + // Mark this download as having an error. + private void error() { + status = ERROR; + stateChanged(); + } + + // Mark this download as having an error. + private void error(String message) { + this.errorMessage = message; + error(); + } + + // Start or resume downloading. + private void download() { +// Thread thread = new Thread(this); +// thread.start(); + run(); + } + + // Get file name portion of URL. + private String getFileName(URL url) { + String result = null; + + String destDir = this.getDestDir(); + String fileName = url.getFile(); + fileName = fileName.substring(fileName.lastIndexOf('/') + 1); + result = filePath = destDir + fileName; + + return result; + } + + // Download file. + public void run() { + RandomAccessFile file = null; + InputStream stream = null; + + try { + // Open connection to URL. + HttpURLConnection connection = + (HttpURLConnection) url.openConnection(); + + // Specify what portion of file to download. + connection.setRequestProperty("Range", + "bytes=" + downloaded + "-"); + + // Connect to server. + connection.connect(); + + // Make sure response code is in the 200 range. + int responseCode = connection.getResponseCode(); + if (responseCode / 100 != 2) { + error(); + } + + // Check for valid content length. + int contentLength = connection.getContentLength(); + if (contentLength < 1) { + error(); + } + + /* Set the size for this download if it + hasn't been already set. */ + if (size == -1) { + size = contentLength; + stateChanged(); + } + + // Open file and seek to the end of it. + File javaFile = new File(getFileName(url)); + file = new RandomAccessFile(javaFile, "rw"); + file.seek(downloaded); + + stream = connection.getInputStream(); + while (status == DOWNLOADING) { + /* Size buffer according to how much of the + file is left to download. */ + byte buffer[]; + if (size - downloaded > MAX_BUFFER_SIZE) { + buffer = new byte[MAX_BUFFER_SIZE]; + } else { + buffer = new byte[size - downloaded]; + } + + // Read from server into buffer. + int read = stream.read(buffer); + if (read == -1) + break; + + // Write buffer to file. + file.write(buffer, 0, read); + downloaded += read; + stateChanged(); + } + + /* Change status to complete if this point was + reached because downloading has finished. */ + if (status == DOWNLOADING) { + status = COMPLETE; + downloadedFile = new File(this.getFilePath()); + stateChanged(); + } + } catch (Exception e) { + e.printStackTrace(); + error(); + } finally { + // Close file. + if (file != null) { + try { + file.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + // Close connection to server. + if (stream != null) { + try { + stream.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + } + + // Notify observers that this download's status has changed. + private void stateChanged() { + setChanged(); + notifyObservers(); + } +} \ No newline at end of file diff --git a/services/common/src/main/java/org/collectionspace/services/common/FileUtils.java b/services/common/src/main/java/org/collectionspace/services/common/FileUtils.java index 6c85676af..5d085a182 100644 --- a/services/common/src/main/java/org/collectionspace/services/common/FileUtils.java +++ b/services/common/src/main/java/org/collectionspace/services/common/FileUtils.java @@ -34,9 +34,9 @@ import java.util.UUID; //import java.io.IOException; -import java.util.UUID; -import java.util.regex.Pattern; -import java.util.regex.Matcher; +//import java.util.UUID; +//import java.util.regex.Pattern; +//import java.util.regex.Matcher; //import javax.servlet.ServletException; //import javax.servlet.http.HttpServlet; @@ -129,13 +129,30 @@ public class FileUtils { if (fileName == null) { fileName = DEFAULT_BLOB_NAME; //if there's no file name then set it to an empty string logger.warn("File was posted to the services without a file name."); - } + } + // + // To avoid name collisions and to preserve the posted file name, create a temp directory for the + // file. + // File tmpDir = new File(System.getProperty("java.io.tmpdir")); - File savedFile = File.createTempFile(TMP_BLOB_PREFIX, fileName, tmpDir); - - item.write(savedFile); + String fileTmpDirPath = tmpDir.getPath() + File.separatorChar + UUID.randomUUID(); + File fileTmpDir = new File(fileTmpDirPath); + File savedFile = null; + if (fileTmpDir.mkdir() == true) { + savedFile = new File(fileTmpDirPath + File.separatorChar + fileName); + if (savedFile.createNewFile() == false) { + savedFile = null; + } + } + + if (savedFile != null) { + item.write(savedFile); // item.getInputStream();//FIXME: REM - We should make a version of this method that returns the input stream - result = savedFile; + result = savedFile; + } else { + logger.error("Could not create temporary file: " + fileTmpDirPath + + File.separatorChar + fileName); + } } } } catch (Exception e) { diff --git a/services/common/src/main/java/org/collectionspace/services/common/blob/BlobInput.java b/services/common/src/main/java/org/collectionspace/services/common/blob/BlobInput.java index 70b871aff..56f06f11f 100644 --- a/services/common/src/main/java/org/collectionspace/services/common/blob/BlobInput.java +++ b/services/common/src/main/java/org/collectionspace/services/common/blob/BlobInput.java @@ -2,15 +2,26 @@ package org.collectionspace.services.common.blob; import java.io.File; import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.servlet.http.HttpServletRequest; //import org.collectionspace.services.blob.BlobsCommonList; -import org.collectionspace.services.jaxb.AbstractCommonList; +//import org.collectionspace.services.jaxb.AbstractCommonList; import org.collectionspace.services.nuxeo.client.java.CommonList; -import org.collectionspace.services.common.FileUtils; +//import org.collectionspace.services.blob.nuxeo.BlobDocumentModelHandler; +//import org.collectionspace.services.common.FileUtils; +import org.collectionspace.services.common.Download; +import org.slf4j.LoggerFactory; +import org.apache.commons.io.FileUtils; public class BlobInput { + private final Logger logger = LoggerFactory.getLogger(BlobInput.class); + private String blobCsid = null; private File blobFile = null; private String blobUri = null; @@ -117,14 +128,35 @@ public class BlobInput { * End of setters and getters */ + // + // FIXME: REM - The callers of this method are sending us a multipart form-data post, so why + // are we also receiving the blobUri? + // public void createBlobFile(HttpServletRequest req, String blobUri) { - File tmpFile = FileUtils.createTmpFile(req); + File tmpFile = org.collectionspace.services.common.FileUtils.createTmpFile(req); this.setBlobFile(tmpFile); this.setBlobUri(blobUri); } - public void createBlobFile(String theBlobUri) { - File theBlobFile = new File(theBlobUri); + public void createBlobFile(String theBlobUri) throws MalformedURLException, Exception { + URL blobUrl = new URL(theBlobUri); + File theBlobFile = null; + + if (blobUrl.getProtocol().equalsIgnoreCase("http")) { + Download fetchedFile = new Download(blobUrl); + while (fetchedFile.getStatus() == Download.DOWNLOADING) { + // Do nothing while we wait for the file to download + } + int status = fetchedFile.getStatus(); + if (status == Download.COMPLETE) { + theBlobFile = fetchedFile.getFile(); + } + } else if (blobUrl.getProtocol().equalsIgnoreCase("file")) { + theBlobFile = FileUtils.toFile(blobUrl); +// theBlobFile = new File(theBlobUri); + } else { + + } this.setBlobFile(theBlobFile); this.setBlobUri(blobUri); } diff --git a/services/jaxb/src/main/resources/blobs_common.xsd b/services/jaxb/src/main/resources/blobs_common.xsd index 8dc7186de..1ac4adc5b 100644 --- a/services/jaxb/src/main/resources/blobs_common.xsd +++ b/services/jaxb/src/main/resources/blobs_common.xsd @@ -50,6 +50,7 @@ maxOccurs="unbounded"/> + diff --git a/services/media/client/src/test/java/org/collectionspace/services/client/test/MediaServiceTest.java b/services/media/client/src/test/java/org/collectionspace/services/client/test/MediaServiceTest.java index a3d87e112..3253d4bda 100644 --- a/services/media/client/src/test/java/org/collectionspace/services/client/test/MediaServiceTest.java +++ b/services/media/client/src/test/java/org/collectionspace/services/client/test/MediaServiceTest.java @@ -23,6 +23,7 @@ package org.collectionspace.services.client.test; import java.io.File; +import java.net.URL; import java.util.List; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; @@ -157,7 +158,8 @@ public class MediaServiceTest extends AbstractServiceTestImpl { logger.debug("Processing file URI: " + blobFile.getAbsolutePath()); logger.debug("MIME type is: " + mimeType); if (fromUri == true) { - res = client.createBlobFromUri(mediaCsid, blobFile.getAbsolutePath()); + URL childUrl = blobFile.toURI().toURL(); + res = client.createBlobFromUri(mediaCsid, childUrl.toString()); } else { MultipartFormDataOutput formData = new MultipartFormDataOutput(); OutputPart outputPart = formData.addFormData("file", blobFile, MediaType.valueOf(mimeType)); -- 2.47.3