dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #40494
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 20543: File store: where applicable (external BlobStore such as AWS S3) GET to dataValues/files now retu...
------------------------------------------------------------
revno: 20543
committer: Halvdan Hoem Grelland <halvdanhg@xxxxxxxxx>
branch nick: dhis2
timestamp: Wed 2015-10-07 00:00:49 +0200
message:
File store: where applicable (external BlobStore such as AWS S3) GET to dataValues/files now returns a redirect with a signed, time limited GET request directly to the external store. Greatly reduces server load and latency when using a third party storage provider. The signed requests are statically limited to a 5 minute life span.
modified:
dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceContentStore.java
dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceService.java
dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceService.java
dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/JCloudsFileResourceContentStore.java
dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataValueController.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceContentStore.java'
--- dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceContentStore.java 2015-09-21 14:33:06 +0000
+++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceContentStore.java 2015-10-06 22:00:49 +0000
@@ -30,6 +30,8 @@
import com.google.common.io.ByteSource;
+import java.net.URI;
+
/**
* @author Halvdan Hoem Grelland
*/
@@ -57,4 +59,11 @@
* @param key the key.
*/
void deleteFileResourceContent( String key );
+
+ /**
+ * Create a signed GET request which gives access to the content.
+ * @param key the key.
+ * @return a URI containing the signed GET request or null if signed requests are not supported.
+ */
+ URI getSignedGetContentUri( String key );
}
=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceService.java'
--- dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceService.java 2015-10-02 22:42:59 +0000
+++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceService.java 2015-10-06 22:00:49 +0000
@@ -30,6 +30,7 @@
import com.google.common.io.ByteSource;
+import java.net.URI;
import java.util.List;
/**
@@ -50,4 +51,6 @@
boolean fileResourceExists( String uid );
void updateFileResource( FileResource fileResource );
+
+ URI getSignedGetFileResourceContentUri( String uid );
}
=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceService.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceService.java 2015-10-02 22:42:59 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceService.java 2015-10-06 22:00:49 +0000
@@ -36,6 +36,7 @@
import com.google.common.io.ByteSource;
+import java.net.URI;
import java.util.List;
/**
@@ -144,6 +145,19 @@
fileResourceStore.update( fileResource );
}
+ @Override
+ public URI getSignedGetFileResourceContentUri( String uid )
+ {
+ FileResource fileResource = getFileResource( uid );
+
+ if ( fileResource == null )
+ {
+ return null;
+ }
+
+ return fileResourceContentStore.getSignedGetContentUri( getRelativeStorageKey( fileResource ) );
+ }
+
// ---------------------------------------------------------------------
// Supportive methods
// ---------------------------------------------------------------------
=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/JCloudsFileResourceContentStore.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/JCloudsFileResourceContentStore.java 2015-10-01 09:45:57 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/JCloudsFileResourceContentStore.java 2015-10-06 22:00:49 +0000
@@ -37,15 +37,21 @@
import org.hisp.dhis.external.location.LocationManager;
import org.hisp.dhis.hibernate.HibernateConfigurationProvider;
import org.jclouds.ContextBuilder;
+import org.jclouds.blobstore.BlobRequestSigner;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.BlobStoreContext;
+import org.jclouds.blobstore.LocalBlobRequestSigner;
import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.internal.RequestSigningUnsupported;
import org.jclouds.domain.Credentials;
import org.jclouds.domain.Location;
import org.jclouds.filesystem.reference.FilesystemConstants;
+import org.jclouds.http.HttpRequest;
+import org.joda.time.Minutes;
import java.io.IOException;
import java.io.InputStream;
+import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -65,6 +71,8 @@
private static final Pattern CONTAINER_NAME_PATTERN = Pattern.compile( "^((?!-)[a-zA-Z0-9-]{1,63}(?<!-))+$" );
+ private static final long FIVE_MINUTES_IN_SECONDS = Minutes.minutes( 5 ).toStandardDuration().getStandardSeconds();
+
private BlobStore blobStore;
private BlobStoreContext blobStoreContext;
private String container;
@@ -202,6 +210,7 @@
// FileResourceContentStore implementation
// -------------------------------------------------------------------------
+ @Override
public ByteSource getFileResourceContent( String key )
{
final Blob blob = getBlob( key );
@@ -241,6 +250,7 @@
return isEmptyOrFailed ? null : byteSource;
}
+ @Override
public String saveFileResourceContent( String key, ByteSource content, long size, String contentMd5 )
{
Blob blob = createBlob( key, content, size, contentMd5 );
@@ -255,11 +265,36 @@
return key;
}
+ @Override
public void deleteFileResourceContent( String key )
{
deleteBlob( key );
}
+ @Override
+ public URI getSignedGetContentUri( String key )
+ {
+ BlobRequestSigner signer = blobStoreContext.getSigner();
+
+ if ( !requestSigningSupported( signer ) )
+ {
+ return null;
+ }
+
+ HttpRequest httpRequest = null;
+
+ try
+ {
+ httpRequest = signer.signGetBlob( container, key, FIVE_MINUTES_IN_SECONDS );
+ }
+ catch ( UnsupportedOperationException uoe )
+ {
+ return null;
+ }
+
+ return httpRequest.getEndpoint();
+ }
+
// -------------------------------------------------------------------------
// Supportive methods
// -------------------------------------------------------------------------
@@ -320,4 +355,9 @@
{
return containerName != null && CONTAINER_NAME_PATTERN.matcher( containerName ).matches();
}
+
+ private boolean requestSigningSupported( BlobRequestSigner signer )
+ {
+ return !( signer instanceof RequestSigningUnsupported ) && !( signer instanceof LocalBlobRequestSigner );
+ }
}
=== modified file 'dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataValueController.java'
--- dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataValueController.java 2015-10-06 19:48:48 +0000
+++ dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataValueController.java 2015-10-06 22:00:49 +0000
@@ -30,6 +30,7 @@
import java.io.IOException;
import java.io.InputStream;
+import java.net.URI;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@@ -470,6 +471,23 @@
response.setContentLength( Math.round( fileResource.getContentLength() ) );
response.setHeader( HttpHeaders.CONTENT_DISPOSITION, "filename=" + fileResource.getName() );
+ // ---------------------------------------------------------------------
+ // Attempt to build signed URL request for content and redirect
+ // ---------------------------------------------------------------------
+
+ URI signedGetUri = fileResourceService.getSignedGetFileResourceContentUri( uid );
+
+ if ( signedGetUri != null )
+ {
+ response.setStatus( HttpServletResponse.SC_FOUND );
+ response.setHeader( HttpHeaders.LOCATION, signedGetUri.toASCIIString() );
+ return;
+ }
+
+ // ---------------------------------------------------------------------
+ // Request signing is not available, stream content back to client
+ // ---------------------------------------------------------------------
+
InputStream inputStream = null;
try
