dhis2-devs team mailing list archive

[noreply@xxxxxxxxxxxxx]

------------------------------------------------------------
revno: 20677
committer: Halvdan Hoem Grelland <halvdanhg@xxxxxxxxx>
branch nick: dhis2
timestamp: Wed 2015-10-14 00:07:31 +0200
message:
  Added scheduled task which purges any orphaned FileResources. A FileResource is considered an orphan if it is created more than two hours ago and is not STORED or not assigned (implying storage or the content failed or the object was never tied to a data value).
added:
  dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/FileResourceCleanUpTask.java
modified:
  dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/GenericIdentifiableObjectStore.java
  dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceService.java
  dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/common/hibernate/HibernateIdentifiableObjectStore.java
  dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceService.java
  dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/FileResourceUploadCallbackProvider.java
  dhis-2/dhis-services/dhis-service-core/src/main/resources/META-INF/dhis/beans.xml
  dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/Scheduler.java


--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk

Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription

=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/GenericIdentifiableObjectStore.java'
--- dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/GenericIdentifiableObjectStore.java	2015-10-12 09:16:42 +0000
+++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/GenericIdentifiableObjectStore.java	2015-10-13 22:07:31 +0000
@@ -234,6 +234,14 @@
     List<T> getAllGeCreated( Date created );
 
     /**
+     * Returns all objects which are equal to or older than the given date.
+     *
+     * @param created Date to compare with.
+     * @return All objects equals to or older than the given date.
+     */
+    List<T> getAllLeCreated( Date created );
+
+    /**
      * Returns all objects that are equal to or newer than given date.
      *
      * @param lastUpdated Date to compare with.

=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceService.java'
--- dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceService.java	2015-10-12 18:52:30 +0000
+++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceService.java	2015-10-13 22:07:31 +0000
@@ -43,6 +43,8 @@
 
     List<FileResource> getFileResources( List<String> uids );
 
+    List<FileResource> getOrphanedFileResources();
+
     String saveFileResource( FileResource fileResource, File file );
 
     void deleteFileResource( String uid );

=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/common/hibernate/HibernateIdentifiableObjectStore.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/common/hibernate/HibernateIdentifiableObjectStore.java	2015-10-12 10:12:49 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/common/hibernate/HibernateIdentifiableObjectStore.java	2015-10-13 22:07:31 +0000
@@ -380,6 +380,16 @@
 
     @Override
     @SuppressWarnings( "unchecked" )
+    public List<T> getAllLeCreated( Date created )
+    {
+        return getSharingCriteria()
+            .add( Restrictions.le( "created", created ) )
+            .addOrder( Order.desc( "created" ) )
+            .list();
+    }
+
+    @Override
+    @SuppressWarnings( "unchecked" )
     public List<T> getAllGeCreatedOrderedName( Date created )
     {
         return getSharingCriteria()

=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceService.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceService.java	2015-10-13 13:10:18 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceService.java	2015-10-13 22:07:31 +0000
@@ -31,12 +31,18 @@
 import com.google.common.io.ByteSource;
 import org.hisp.dhis.common.GenericIdentifiableObjectStore;
 import org.hisp.dhis.system.scheduling.Scheduler;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+import org.joda.time.Hours;
 import org.springframework.transaction.annotation.Transactional;
 import org.springframework.util.concurrent.ListenableFuture;
 
+import javax.annotation.PostConstruct;
 import java.io.File;
 import java.net.URI;
 import java.util.List;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 /**
  * @author Halvdan Hoem Grelland
@@ -44,6 +50,13 @@
 public class DefaultFileResourceService
     implements FileResourceService
 {
+    private static final String KEY_FILE_CLEANUP_TASK = "fileResourceCleanupTask";
+
+    private static final Predicate<FileResource> IS_ORPHAN_PREDICATE =
+        ( fr -> !fr.isAssigned() || fr.getStorageStatus() != FileResourceStorageStatus.STORED );
+
+    private static final Duration IS_ORPHAN_TIME_DELTA = Hours.TWO.toStandardDuration();
+
     // -------------------------------------------------------------------------
     // Dependencies
     // -------------------------------------------------------------------------
@@ -76,6 +89,27 @@
         this.uploadCallbackProvider = uploadCallbackProvider;
     }
 
+    private FileResourceCleanUpTask fileResourceCleanUpTask;
+
+    public void setFileResourceCleanUpTask( FileResourceCleanUpTask fileResourceCleanUpTask )
+    {
+        this.fileResourceCleanUpTask = fileResourceCleanUpTask;
+    }
+
+    // -------------------------------------------------------------------------
+    // Init
+    // -------------------------------------------------------------------------
+
+    @PostConstruct
+    public void init()
+    {
+        // Background task which queries for non-assigned or failed-upload
+        // FileResources and deletes them from the database and/or file store.
+        // Runs every day at 2 AM server time.
+
+        scheduler.scheduleTask( KEY_FILE_CLEANUP_TASK, fileResourceCleanUpTask, Scheduler.CRON_DAILY_2AM );
+    }
+
     // -------------------------------------------------------------------------
     // FileResourceService implementation
     // -------------------------------------------------------------------------
@@ -94,6 +128,14 @@
 
     @Transactional
     @Override
+    public List<FileResource> getOrphanedFileResources( )
+    {
+        return fileResourceStore.getAllLeCreated( new DateTime().minus( IS_ORPHAN_TIME_DELTA ).toDate() )
+            .stream().filter( IS_ORPHAN_PREDICATE ).collect( Collectors.toList() );
+    }
+
+    @Transactional
+    @Override
     public String saveFileResource( FileResource fileResource, File file )
     {
         fileResource.setStorageStatus( FileResourceStorageStatus.PENDING );

=== added file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/FileResourceCleanUpTask.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/FileResourceCleanUpTask.java	1970-01-01 00:00:00 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/FileResourceCleanUpTask.java	2015-10-13 22:07:31 +0000
@@ -0,0 +1,86 @@
+package org.hisp.dhis.fileresource;
+
+/*
+ * Copyright (c) 2004-2015, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Deletes any orphaned FileResources.
+ *
+ * @author Halvdan Hoem Grelland
+ */
+public class FileResourceCleanUpTask
+    implements Runnable
+{
+    private static final Log log = LogFactory.getLog( FileResourceCleanUpTask.class );
+
+    @Autowired
+    private FileResourceService fileResourceService;
+
+    @Override
+    public void run()
+    {
+        List<Pair<String, String>> deleted = new ArrayList<>();
+
+        fileResourceService.getOrphanedFileResources()
+            .stream()
+            .forEach( fr -> {
+                deleted.add( ImmutablePair.of( fr.getName(), fr.getUid() ) );
+                fileResourceService.deleteFileResource( fr.getUid() );
+            } );
+
+        if ( !deleted.isEmpty() )
+        {
+            log.warn( "Deleted " + deleted.size() + " orphaned FileResources: " + prettyPrint( deleted ) );
+        }
+    }
+
+    private String prettyPrint( List<Pair<String, String>> list )
+    {
+        if ( list.isEmpty() )
+        {
+            return "";
+        }
+
+        StringBuilder sb = new StringBuilder( "[ " );
+
+        list.forEach( pair -> sb.append( pair.getLeft() ).append( " , uid: " ).append( pair.getRight() ).append( " ," ) );
+
+        sb.deleteCharAt( sb.lastIndexOf( "," ) ).append( "]" );
+
+        return sb.toString();
+    }
+}

=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/FileResourceUploadCallbackProvider.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/FileResourceUploadCallbackProvider.java	2015-10-07 23:01:57 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/FileResourceUploadCallbackProvider.java	2015-10-13 22:07:31 +0000
@@ -61,7 +61,7 @@
             @Override
             public void onSuccess( String result )
             {
-                log.info( "File content uploaded: " + result );
+                log.info( "File content stored: " + result );
 
                 FileResource fetchedFileResource = idObjectManager.get( FileResource.class, fileResourceUid );
 

=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/resources/META-INF/dhis/beans.xml'
--- dhis-2/dhis-services/dhis-service-core/src/main/resources/META-INF/dhis/beans.xml	2015-10-13 13:10:18 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/resources/META-INF/dhis/beans.xml	2015-10-13 22:07:31 +0000
@@ -571,6 +571,11 @@
     <property name="sessionFactory" ref="sessionFactory" />
   </bean>
 
+  <bean id="org.hisp.dhis.keyjsonvalue.KeyJsonValueStore" class="org.hisp.dhis.keyjsonvalue.hibernate.HibernateKeyJsonValueStore">
+    <property name="clazz" value="org.hisp.dhis.keyjsonvalue.KeyJsonValue" />
+    <property name="sessionFactory" ref="sessionFactory" />
+  </bean>
+
   <bean id="org.hisp.dhis.fileresource.FileResourceStore" class="org.hisp.dhis.common.hibernate.HibernateIdentifiableObjectStore">
     <property name="clazz" value="org.hisp.dhis.fileresource.FileResource" />
     <property name="sessionFactory" ref="sessionFactory" />
@@ -582,14 +587,15 @@
     <property name="locationManager" ref="locationManager" />
   </bean>
 
+  <!-- File resource tasks -->
+
   <bean id="org.hisp.dhis.fileresource.FileResourceUploadCallbackProvider"
     class="org.hisp.dhis.fileresource.FileResourceUploadCallbackProvider"
     scope="prototype" />
 
-  <bean id="org.hisp.dhis.keyjsonvalue.KeyJsonValueStore" class="org.hisp.dhis.keyjsonvalue.hibernate.HibernateKeyJsonValueStore">
-    <property name="clazz" value="org.hisp.dhis.keyjsonvalue.KeyJsonValue" />
-    <property name="sessionFactory" ref="sessionFactory" />
-  </bean>
+  <bean id="org.hisp.dhis.fileresource.FileResourceCleanUpTask"
+    class="org.hisp.dhis.fileresource.FileResourceCleanUpTask"
+    scope="prototype" />
 
   <!-- Service definitions -->
 
@@ -598,6 +604,7 @@
     <property name="fileResourceContentStore" ref="org.hisp.dhis.fileresource.FileResourceContentStore" />
     <property name="scheduler" ref="scheduler" />
     <property name="uploadCallbackProvider" ref="org.hisp.dhis.fileresource.FileResourceUploadCallbackProvider" />
+    <property name="fileResourceCleanUpTask" ref="org.hisp.dhis.fileresource.FileResourceCleanUpTask" />
   </bean>
 
   <bean id="org.hisp.dhis.dataelement.DataElementOperandService" class="org.hisp.dhis.dataelement.DefaultDataElementOperandService">

=== modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/Scheduler.java'
--- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/Scheduler.java	2015-10-07 13:25:14 +0000
+++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/Scheduler.java	2015-10-13 22:07:31 +0000
@@ -41,6 +41,7 @@
 {
     String CRON_DAILY_0AM = "0 0 0 * * ?";
     String CRON_DAILY_11PM = "0 0 23 * * ?";
+    String CRON_DAILY_2AM = "0 0 2 * * ?";
     String CRON_EVERY_MIN = "0 0/1 * * * ?";
     String CRON_EVERY_15MIN = "0 0/15 * * * ?";
     String CRON_TEST = "0 * * * * ?";