dhis2-devs team mailing list archive

[noreply@xxxxxxxxxxxxx]

------------------------------------------------------------
revno: 14578
committer: Morten Olav Hansen <mortenoh@xxxxxxxxx>
branch nick: dhis2
timestamp: Tue 2014-04-01 11:22:49 +0700
message:
  include timer info in event import, optimize import speed by flushing session for every 100th event
modified:
  dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/AbstractEventService.java
  dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/JacksonEventService.java
  dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/api/controller/event/EventController.java


--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk

Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription

=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/AbstractEventService.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/AbstractEventService.java	2014-03-31 08:44:01 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/AbstractEventService.java	2014-04-01 04:22:49 +0000
@@ -225,7 +225,6 @@
                     programStageInstance = programStageInstanceService.getProgramStageInstance( programInstance,
                         programStage );
                 }
-
                 else
                 {
                     if ( event.getEvent() != null )

=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/JacksonEventService.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/JacksonEventService.java	2014-03-18 08:10:10 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/JacksonEventService.java	2014-04-01 04:22:49 +0000
@@ -31,9 +31,14 @@
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.dataformat.xml.XmlMapper;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.hibernate.SessionFactory;
 import org.hisp.dhis.dxf2.importsummary.ImportSummaries;
 import org.hisp.dhis.dxf2.importsummary.ImportSummary;
 import org.hisp.dhis.dxf2.metadata.ImportOptions;
+import org.hisp.dhis.dxf2.timer.SystemNanoTimer;
+import org.hisp.dhis.dxf2.timer.Timer;
 import org.hisp.dhis.scheduling.TaskId;
 import org.hisp.dhis.system.notification.NotificationLevel;
 import org.hisp.dhis.system.notification.Notifier;
@@ -53,9 +58,14 @@
 @Transactional
 public class JacksonEventService extends AbstractEventService
 {
+    private static final Log log = LogFactory.getLog( JacksonEventService.class );
+
     @Autowired
     private Notifier notifier;
 
+    @Autowired
+    private SessionFactory sessionFactory;
+
     // -------------------------------------------------------------------------
     // EventService Impl
     // -------------------------------------------------------------------------
@@ -64,25 +74,25 @@
 
     private final static ObjectMapper jsonMapper = new ObjectMapper();
 
-    @SuppressWarnings("unchecked")
+    @SuppressWarnings( "unchecked" )
     private static <T> T fromXml( InputStream inputStream, Class<?> clazz ) throws IOException
     {
         return (T) xmlMapper.readValue( inputStream, clazz );
     }
 
-    @SuppressWarnings("unchecked")
+    @SuppressWarnings( "unchecked" )
     private static <T> T fromXml( String input, Class<?> clazz ) throws IOException
     {
         return (T) xmlMapper.readValue( input, clazz );
     }
 
-    @SuppressWarnings("unchecked")
+    @SuppressWarnings( "unchecked" )
     private static <T> T fromJson( InputStream inputStream, Class<?> clazz ) throws IOException
     {
         return (T) jsonMapper.readValue( inputStream, clazz );
     }
 
-    @SuppressWarnings("unchecked")
+    @SuppressWarnings( "unchecked" )
     private static <T> T fromJson( String input, Class<?> clazz ) throws IOException
     {
         return (T) jsonMapper.readValue( input, clazz );
@@ -119,13 +129,26 @@
 
         notifier.clear( taskId ).notify( taskId, "Importing events" );
 
+        Timer timer = new SystemNanoTimer();
+        timer.start();
+
         try
         {
             Events events = fromXml( input, Events.class );
 
+            int counter = 0;
+
             for ( Event event : events.getEvents() )
             {
                 importSummaries.addImportSummary( addEvent( event, importOptions ) );
+
+                if ( counter % 100 == 0 )
+                {
+                    sessionFactory.getCurrentSession().flush();
+                    sessionFactory.getCurrentSession().clear();
+                }
+
+                counter++;
             }
         }
         catch ( Exception ex )
@@ -134,8 +157,17 @@
             importSummaries.addImportSummary( addEvent( event, importOptions ) );
         }
 
-        notifier.notify( taskId, NotificationLevel.INFO, "Import done", true ).
-            addTaskSummary( taskId, importSummaries );
+        timer.stop();
+
+        if ( taskId != null )
+        {
+            notifier.notify( taskId, NotificationLevel.INFO, "Import done. Completed in " + timer.toString() + ".", true ).
+                addTaskSummary( taskId, importSummaries );
+        }
+        else
+        {
+            log.info( "Import done. Completed in " + timer.toString() + "." );
+        }
 
         return importSummaries;
     }
@@ -174,13 +206,26 @@
 
         notifier.clear( taskId ).notify( taskId, "Importing events" );
 
+        Timer timer = new SystemNanoTimer();
+        timer.start();
+
         try
         {
             Events events = fromJson( input, Events.class );
 
+            int counter = 0;
+
             for ( Event event : events.getEvents() )
             {
                 importSummaries.addImportSummary( addEvent( event, importOptions ) );
+
+                if ( counter % 100 == 0 )
+                {
+                    sessionFactory.getCurrentSession().flush();
+                    sessionFactory.getCurrentSession().clear();
+                }
+
+                counter++;
             }
         }
         catch ( Exception ex )
@@ -189,8 +234,17 @@
             importSummaries.addImportSummary( addEvent( event, importOptions ) );
         }
 
-        notifier.notify( taskId, NotificationLevel.INFO, "Import done", true ).
-            addTaskSummary( taskId, importSummaries );
+        timer.stop();
+
+        if ( taskId != null )
+        {
+            notifier.notify( taskId, NotificationLevel.INFO, "Import done. Completed in " + timer.toString() + ".", true ).
+                addTaskSummary( taskId, importSummaries );
+        }
+        else
+        {
+            log.info( "Import done. Completed in " + timer.toString() + "." );
+        }
 
         return importSummaries;
     }

=== modified file 'dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/api/controller/event/EventController.java'
--- dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/api/controller/event/EventController.java	2014-03-28 15:54:51 +0000
+++ dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/api/controller/event/EventController.java	2014-04-01 04:22:49 +0000
@@ -338,7 +338,6 @@
             response.setHeader( "Location", ContextUtils.getRootPath( request ) + "/system/tasks/" + TaskCategory.EVENT_IMPORT );
             response.setStatus( HttpServletResponse.SC_NO_CONTENT );
         }
-
     }
 
     // -------------------------------------------------------------------------