dhis2-devs team mailing list archive

[noreply@xxxxxxxxxxxxx]

------------------------------------------------------------
revno: 3417
committer: Lars Helge Overland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Sat 2011-04-16 18:18:58 +0200
message:
  Crosstab step/aggregation in datamart process now runs agains an in-memory HSQL database. This means that we are not prohibited by fixed restrictions on number of columns in the crosstab table. Makes performance more scalable in terms of available memory and less cpu/disk read-write bound. Cuts time with 40% on kenya nigtly export.
modified:
  dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml
  dhis-2/dhis-support/dhis-support-jdbc/src/main/resources/META-INF/dhis/beans.xml
  dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/process/AbstractStatementInternalProcess.java
  dhis-2/pom.xml


--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk

Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription

=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml	2011-04-14 09:05:18 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml	2011-04-16 16:18:58 +0000
@@ -70,6 +70,7 @@
     class="org.hisp.dhis.datamart.DataMartInternalProcess"
     scope="prototype">
     <property name="statementManager" ref="statementManager"/>
+    <property name="inMemoryStatementManager" ref="inMemoryStatementManager"/>
     <property name="dataMartService"
       ref="org.hisp.dhis.datamart.DataMartService"/>
   </bean>
@@ -78,12 +79,12 @@
   
   <bean id="org.hisp.dhis.datamart.crosstab.jdbc.CrossTabStore"
     class="org.hisp.dhis.datamart.crosstab.jdbc.JDBCCrossTabStore">
-    <property name="statementManager" ref="statementManager"/>
+    <property name="statementManager" ref="inMemoryStatementManager"/>
   </bean>
   
   <bean id="org.hisp.dhis.datamart.crosstab.CrossTabService"
     class="org.hisp.dhis.datamart.crosstab.DefaultCrossTabService">
-    <property name="batchHandlerFactory" ref="batchHandlerFactory"/>
+    <property name="batchHandlerFactory" ref="inMemoryBatchHandlerFactory"/>
     <property name="crossTabStore"
       ref="org.hisp.dhis.datamart.crosstab.jdbc.CrossTabStore"/>
     <property name="aggregatedDataValueService"
@@ -223,7 +224,7 @@
       <aop:before pointcut="execution( * org.hisp.dhis.datamart.DataMartService.deleteDataMartExport(..) )" method="intercept"/>
     </aop:aspect>
 
-    <aop:aspect ref="statementInterceptor">
+    <aop:aspect ref="inMemoryStatementInterceptor">
       <aop:around pointcut="execution( * org.hisp.dhis.datamart.DataMartService.export(..) )" method="intercept" />
 	</aop:aspect>
       

=== modified file 'dhis-2/dhis-support/dhis-support-jdbc/src/main/resources/META-INF/dhis/beans.xml'
--- dhis-2/dhis-support/dhis-support-jdbc/src/main/resources/META-INF/dhis/beans.xml	2011-04-15 14:38:27 +0000
+++ dhis-2/dhis-support/dhis-support-jdbc/src/main/resources/META-INF/dhis/beans.xml	2011-04-16 16:18:58 +0000
@@ -10,16 +10,37 @@
     <property name="jdbcConfiguration" ref="jdbcConfiguration"/>
   </bean>
   
+  <bean id="inMemoryStatementManager" class="org.amplecode.quick.statement.JdbcStatementManager">
+    <property name="inMemory" value="true"/>
+  </bean>
+  
   <!-- BatchHandlerFactory -->
     
   <bean id="batchHandlerFactory" class="org.amplecode.quick.factory.DefaultBatchHandlerFactory">
     <property name="jdbcConfiguration" ref="jdbcConfiguration"/>
   </bean>
-
+  
+  <bean id="inMemoryBatchHandlerFactory" class="org.amplecode.quick.factory.DefaultBatchHandlerFactory">
+    <property name="inMemory" value="true"/>
+  </bean>
+  
   <!-- Interceptor -->
     
   <bean id="statementInterceptor" class="org.amplecode.quick.StatementInterceptor">
-    <property name="statementManager" ref="statementManager"/>
+    <property name="statementManagers">
+      <list>
+        <ref bean="statementManager"/>
+      </list>
+    </property>
+  </bean>
+
+  <bean id="inMemoryStatementInterceptor" class="org.amplecode.quick.StatementInterceptor">
+    <property name="statementManagers">
+      <list>
+        <ref bean="statementManager"/>
+        <ref bean="inMemoryStatementManager"/>
+      </list>
+    </property>
   </bean>
 
   <!-- StatementDialect -->

=== modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/process/AbstractStatementInternalProcess.java'
--- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/process/AbstractStatementInternalProcess.java	2010-07-02 14:03:28 +0000
+++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/process/AbstractStatementInternalProcess.java	2011-04-16 16:18:58 +0000
@@ -64,6 +64,13 @@
         this.statementManager = statementManager;
     }
     
+    private StatementManager inMemoryStatementManager;
+
+    public void setInMemoryStatementManager( StatementManager inMemoryStatementManager )
+    {
+        this.inMemoryStatementManager = inMemoryStatementManager;
+    }
+
     // -------------------------------------------------------------------------
     // InternalProcess implementation
     // -------------------------------------------------------------------------
@@ -82,6 +89,11 @@
 
         statementManager.initialise();
         
+        if ( inMemoryStatementManager != null )
+        {
+            inMemoryStatementManager.initialise();
+        }
+        
         getState().setMessage( PROCESS_STARTED );
         
         log.info( "Internal process started" );
@@ -106,6 +118,11 @@
         finally
         {   
             statementManager.destroy();
+            
+            if ( inMemoryStatementManager != null )
+            {
+                inMemoryStatementManager.destroy();
+            }
         }
     }
 

=== modified file 'dhis-2/pom.xml'
--- dhis-2/pom.xml	2011-04-15 14:38:27 +0000
+++ dhis-2/pom.xml	2011-04-16 16:18:58 +0000
@@ -379,7 +379,7 @@
       <dependency>
         <groupId>org.amplecode</groupId>
         <artifactId>quick</artifactId>
-        <version>1.4</version>
+        <version>1.5</version>
       </dependency>
       <dependency>
         <groupId>com.lowagie</groupId>