dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #39979
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 20239: Fixed context without environment set up (i.e. tests) tripping up DefaultFileResourceContentStore...
------------------------------------------------------------
revno: 20239
committer: Halvdan Hoem Grelland <halvdanhg@xxxxxxxxx>
branch nick: dhis2
timestamp: Fri 2015-09-18 23:24:50 +0200
message:
Fixed context without environment set up (i.e. tests) tripping up DefaultFileResourceContentStore.init(). Now falling back to file system with no file store config set, and falling back to transient provider if no external directory is set up as well.
removed:
dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/BaseJCloudsFileResourceContentStore.java
modified:
dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceContentStore.java
dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceService.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== removed file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/BaseJCloudsFileResourceContentStore.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/BaseJCloudsFileResourceContentStore.java 2015-09-18 18:01:58 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/BaseJCloudsFileResourceContentStore.java 1970-01-01 00:00:00 +0000
@@ -1,183 +0,0 @@
-package org.hisp.dhis.fileresource;
-
-/*
- * Copyright (c) 2004-2015, University of Oslo
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- * Neither the name of the HISP project nor the names of its contributors may
- * be used to endorse or promote products derived from this software without
- * specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
- * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
- * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Optional;
-import java.util.Properties;
-
-import org.apache.commons.io.input.NullInputStream;
-import org.jclouds.ContextBuilder;
-import org.jclouds.blobstore.BlobStore;
-import org.jclouds.blobstore.BlobStoreContext;
-import org.jclouds.blobstore.domain.Blob;
-import org.jclouds.domain.Credentials;
-import org.jclouds.domain.Location;
-
-import com.google.common.hash.HashCode;
-import com.google.common.io.ByteSource;
-
-/**
- * @author Halvdan Hoem Grelland
- */
-public abstract class BaseJCloudsFileResourceContentStore
- implements FileResourceContentStore
-{
- private BlobStore blobStore;
- private BlobStoreContext blobStoreContext;
-
- // -------------------------------------------------------------------------
- // Default config implementations
- // -------------------------------------------------------------------------
-
- protected Credentials getCredentials()
- {
- return new Credentials( "Unused", "Unused" );
- }
-
- protected Properties getOverrides()
- {
- return new Properties();
- }
-
- // -------------------------------------------------------------------------
- // Abstract methods
- // -------------------------------------------------------------------------
-
- protected abstract String getContainer();
-
- protected abstract String getJCloudsProviderKey();
-
- protected abstract String getLocation();
-
- // -------------------------------------------------------------------------
- // Life cycle management
- // -------------------------------------------------------------------------
-
- public void init()
- {
- Credentials credentials = getCredentials();
- Properties overrides = getOverrides();
- String providerKey = getJCloudsProviderKey();
-
- blobStoreContext = ContextBuilder.newBuilder( providerKey )
- .credentials( credentials.identity, credentials.credential )
- .overrides( overrides ).build( BlobStoreContext.class );
-
- blobStore = blobStoreContext.getBlobStore();
-
- Optional<? extends Location> location = blobStore.listAssignableLocations()
- .stream().filter( l -> l.getId().equals( getLocation() ) ).findFirst();
-
- blobStore.createContainerInLocation( location.isPresent() ? location.get() : null, getContainer() );
- }
-
- public void cleanUp()
- {
- blobStoreContext.close();
- }
-
- // -------------------------------------------------------------------------
- // FileResourceContentStore implementation
- // -------------------------------------------------------------------------
-
- public ByteSource getFileResourceContent( String key )
- {
- final Blob blob = getBlob( key );
-
- if ( blob == null )
- {
- return null;
- }
-
- return new ByteSource()
- {
- @Override
- public InputStream openStream()
- {
- try
- {
- return blob.getPayload().openStream();
- }
- catch ( IOException e )
- {
- return new NullInputStream( 0 );
- }
- }
- };
- }
-
- public String saveFileResourceContent( String key, ByteSource content, long size, String contentMd5 )
- {
- Blob blob = createBlob( key, content, size, contentMd5 );
-
- if ( blob == null )
- {
- return null;
- }
-
- putBlob( blob );
-
- return key;
- }
-
- public void deleteFileResourceContent( String key )
- {
- deleteBlob( key );
- }
-
- // -------------------------------------------------------------------------
- // Supportive methods
- // -------------------------------------------------------------------------
-
- private Blob getBlob( String key )
- {
- return blobStore.getBlob( getContainer(), key );
- }
-
- private void deleteBlob( String key )
- {
- blobStore.removeBlob( getContainer(), key );
- }
-
- private String putBlob( Blob blob )
- {
- return blobStore.putBlob( getContainer(), blob );
- }
-
- private Blob createBlob( String key, ByteSource content, long size, String contentMd5 )
- {
- return blobStore.blobBuilder( key )
- .payload( content )
- .contentLength( size )
- .contentMD5( HashCode.fromString( contentMd5 ) )
- .build();
- }
-}
=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceContentStore.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceContentStore.java 2015-09-18 17:50:30 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceContentStore.java 2015-09-18 21:24:50 +0000
@@ -28,39 +28,57 @@
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
+import com.google.common.hash.HashCode;
+import com.google.common.io.ByteSource;
+import org.apache.commons.io.input.NullInputStream;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.hisp.dhis.external.location.LocationManager;
import org.hisp.dhis.hibernate.HibernateConfigurationProvider;
+import org.jclouds.ContextBuilder;
+import org.jclouds.blobstore.BlobStore;
+import org.jclouds.blobstore.BlobStoreContext;
+import org.jclouds.blobstore.domain.Blob;
import org.jclouds.domain.Credentials;
+import org.jclouds.domain.Location;
import org.jclouds.filesystem.reference.FilesystemConstants;
+import java.io.IOException;
+import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Properties;
import java.util.stream.Collectors;
/**
- * TODO Merge with BaseJCloudsFileResourceContentStore ?
* @author Halvdan Hoem Grelland
*/
public class DefaultFileResourceContentStore
- extends BaseJCloudsFileResourceContentStore
+ implements FileResourceContentStore
{
private static final Log log = LogFactory.getLog( DefaultFileResourceContentStore.class );
+ private BlobStore blobStore;
+ private BlobStoreContext blobStoreContext;
+ private String container;
+
// -------------------------------------------------------------------------
- // Provider constants
+ // Providers
// -------------------------------------------------------------------------
private static final String JCLOUDS_PROVIDER_KEY_FILESYSTEM = "filesystem";
private static final String JCLOUDS_PROVIDER_KEY_AWS_S3 = "aws-s3";
+ private static final String JCLOUDS_PROVIDER_KEY_TRANSIENT = "transient";
- private static final List<String> AVAILABLE_PROVIDERS = new ArrayList<String>() {{
- addAll( Arrays.asList( JCLOUDS_PROVIDER_KEY_FILESYSTEM, JCLOUDS_PROVIDER_KEY_AWS_S3 ) );
+ private static final List<String> SUPPORTED_PROVIDERS = new ArrayList<String>() {{
+ addAll( Arrays.asList(
+ JCLOUDS_PROVIDER_KEY_FILESYSTEM,
+ JCLOUDS_PROVIDER_KEY_AWS_S3
+ ) );
}};
// -------------------------------------------------------------------------
@@ -79,22 +97,9 @@
// Defaults
// -------------------------------------------------------------------------
- private static final String DEFAULT_PROVIDER = "filesystem";
private static final String DEFAULT_CONTAINER = "dhis2_filestore";
// -------------------------------------------------------------------------
- // Configuration
- // -------------------------------------------------------------------------
-
- private Map<String, String> filestoreConfiguration;
-
- private String provider;
- private String container;
- private Credentials credentials;
- private String location;
- private Properties overrides = new Properties();
-
- // -------------------------------------------------------------------------
// Dependencies
// -------------------------------------------------------------------------
@@ -113,115 +118,163 @@
}
// -------------------------------------------------------------------------
- // Lifecycle
+ // Life cycle management
// -------------------------------------------------------------------------
+ // TODO Untangle and split up
public void init()
{
+ // -------------------------------------------------------------------------
+ // Parse properties
+ // -------------------------------------------------------------------------
+
Properties properties = configurationProvider.getConfiguration().getProperties();
-
- filestoreConfiguration = properties
- .entrySet().stream().filter(
- p -> ( (String) p.getKey() ).startsWith( FILESTORE_CONFIG_NAMESPACE ) )
+
+ Map<String, String> filestoreConfiguration = properties
+ .entrySet().stream().filter( p -> ((String) p.getKey()).startsWith( FILESTORE_CONFIG_NAMESPACE ) )
.collect( Collectors.toMap(
p -> StringUtils.strip( (String) p.getKey() ),
p -> StringUtils.strip( (String) p.getValue() )
) );
- provider = filestoreConfiguration.getOrDefault( KEY_FILESTORE_PROVIDER, DEFAULT_PROVIDER );
-
- if ( !AVAILABLE_PROVIDERS.contains( provider ) )
- {
- log.info( "Ignored unsupported file store provider '" + provider + "', falling back to file system." );
- provider = DEFAULT_PROVIDER;
+ String provider = filestoreConfiguration.getOrDefault( KEY_FILESTORE_PROVIDER, JCLOUDS_PROVIDER_KEY_FILESYSTEM );
+
+ if ( !SUPPORTED_PROVIDERS.contains( provider ) )
+ {
+ log.warn( "Ignored unsupported file store provider '" + provider + "', falling back to file system." );
+ provider = JCLOUDS_PROVIDER_KEY_FILESYSTEM;
+ }
+
+ if ( provider.equals( JCLOUDS_PROVIDER_KEY_FILESYSTEM ) && !locationManager.externalDirectorySet() )
+ {
+ log.warn( "File system store provider configured but external directory is not set. Falling back to transient file store." );
+ provider = JCLOUDS_PROVIDER_KEY_TRANSIENT;
}
container = filestoreConfiguration.getOrDefault( KEY_FILESTORE_CONTAINER, DEFAULT_CONTAINER );
- location = filestoreConfiguration.getOrDefault( KEY_FILESTORE_LOCATION, null );
-
- switch ( provider )
- {
- case JCLOUDS_PROVIDER_KEY_FILESYSTEM:
- configureFilesystemProvider();
- break;
- case JCLOUDS_PROVIDER_KEY_AWS_S3:
- configureAWSS3Provider();
- break;
- default:
- throw new IllegalArgumentException( "The filestore provider " + provider + " is not supported." );
- }
-
- super.init();
+ String location = filestoreConfiguration.getOrDefault( KEY_FILESTORE_LOCATION, null );
+ Properties overrides = new Properties();
+ Credentials credentials = new Credentials( "Unused", "Unused" );
+
+ // -------------------------------------------------------------------------
+ // Provider specific configuration
+ // -------------------------------------------------------------------------
+
+ if ( provider.equals( JCLOUDS_PROVIDER_KEY_FILESYSTEM ) && locationManager.externalDirectorySet() )
+ {
+ overrides.setProperty( FilesystemConstants.PROPERTY_BASEDIR, locationManager.getExternalDirectoryPath() );
+
+ log.info( "File system store provider configured" );
+ }
+ else if ( provider.equals( JCLOUDS_PROVIDER_KEY_AWS_S3 ) )
+ {
+ credentials = new Credentials( filestoreConfiguration.getOrDefault(
+ KEY_FILESTORE_IDENTITY, "" ), filestoreConfiguration.getOrDefault( KEY_FILESTORE_SECRET, "" ) );
+
+ log.info( "AWS S3 filestore provider configured." );
+
+ if ( credentials.identity.isEmpty() || credentials.credential.isEmpty() )
+ {
+ log.info( "AWS S3 store configured with empty credentials, authentication not possible" );
+ }
+ }
+
+ // -------------------------------------------------------------------------
+ // Set up BlobStore
+ // -------------------------------------------------------------------------
+
+ blobStoreContext = ContextBuilder.newBuilder( provider )
+ .credentials( credentials.identity, credentials.credential )
+ .overrides( overrides ).build( BlobStoreContext.class );
+
+ blobStore = blobStoreContext.getBlobStore();
+
+ Optional<? extends Location> configuredLocation = blobStore.listAssignableLocations()
+ .stream().filter( l -> l.getId().equals( location ) ).findFirst();
+
+ blobStore.createContainerInLocation( configuredLocation.isPresent() ? configuredLocation.get() : null, container );
}
public void cleanUp()
{
- super.cleanUp();
- }
-
- // -------------------------------------------------------------------------
- // Configuration implementation
- // -------------------------------------------------------------------------
-
- @Override
- protected Properties getOverrides()
- {
- return overrides;
- }
-
- @Override
- protected Credentials getCredentials()
- {
- return credentials;
- }
-
- @Override
- protected String getContainer()
- {
- return container;
- }
-
- @Override
- protected String getLocation()
- {
- return location;
- }
-
- @Override
- protected String getJCloudsProviderKey()
- {
- return provider;
+ blobStoreContext.close();
+ }
+
+ // -------------------------------------------------------------------------
+ // FileResourceContentStore implementation
+ // -------------------------------------------------------------------------
+
+ public ByteSource getFileResourceContent( String key )
+ {
+ final Blob blob = getBlob( key );
+
+ if ( blob == null )
+ {
+ return null;
+ }
+
+ return new ByteSource()
+ {
+ @Override
+ public InputStream openStream()
+ {
+ try
+ {
+ return blob.getPayload().openStream();
+ }
+ catch ( IOException e )
+ {
+ return new NullInputStream( 0 );
+ }
+ }
+ };
+ }
+
+ public String saveFileResourceContent( String key, ByteSource content, long size, String contentMd5 )
+ {
+ Blob blob = createBlob( key, content, size, contentMd5 );
+
+ if ( blob == null )
+ {
+ return null;
+ }
+
+ putBlob( blob );
+
+ return key;
+ }
+
+ public void deleteFileResourceContent( String key )
+ {
+ deleteBlob( key );
}
// -------------------------------------------------------------------------
// Supportive methods
// -------------------------------------------------------------------------
- private void configureFilesystemProvider()
- {
- if ( locationManager.externalDirectorySet() )
- {
- overrides.setProperty( FilesystemConstants.PROPERTY_BASEDIR, locationManager.getExternalDirectoryPath() );
- credentials = super.getCredentials();
-
- log.info( "File system store provider configured" );
- }
- else
- {
- log.warn( "File system store could not be configured, external directory not set" );
- }
- }
-
- private void configureAWSS3Provider()
- {
- credentials = new Credentials( filestoreConfiguration.getOrDefault(
- KEY_FILESTORE_IDENTITY, "" ), filestoreConfiguration.getOrDefault( KEY_FILESTORE_SECRET, "" ) );
- log.info( "AWS S3 filestore provider configured." );
-
- if ( credentials.identity.isEmpty() || credentials.credential.isEmpty() )
- {
- log.info( "AWS S3 store configured with empty credentials, authentication not possible" );
- }
+ private Blob getBlob( String key )
+ {
+ return blobStore.getBlob( container, key );
+ }
+
+ private void deleteBlob( String key )
+ {
+ blobStore.removeBlob( container, key );
+ }
+
+ private String putBlob( Blob blob )
+ {
+ return blobStore.putBlob( container, blob );
+ }
+
+ private Blob createBlob( String key, ByteSource content, long size, String contentMd5 )
+ {
+ return blobStore.blobBuilder( key )
+ .payload( content )
+ .contentLength( size )
+ .contentMD5( HashCode.fromString( contentMd5 ) )
+ .build();
}
}
=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceService.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceService.java 2015-09-18 20:02:57 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceService.java 2015-09-18 21:24:50 +0000
@@ -33,6 +33,8 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.hisp.dhis.common.GenericIdentifiableObjectStore;
+import org.jclouds.blobstore.BlobStore;
+import org.jclouds.blobstore.BlobStoreContext;
import org.springframework.transaction.annotation.Transactional;
/**
@@ -41,9 +43,12 @@
public class DefaultFileResourceService
implements FileResourceService
{
-
private static final Log log = LogFactory.getLog( DefaultFileResourceService.class );
+ private BlobStore blobStore;
+
+ private BlobStoreContext blobStoreContext;
+
// -------------------------------------------------------------------------
// Dependencies
// -------------------------------------------------------------------------
