dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #14586
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 5001: Data mart, generating the crosstabulated table in parallel processes. Near lineary performance im...
------------------------------------------------------------
revno: 5001
committer: Lars Helge Overland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Sat 2011-10-22 10:41:00 +0200
message:
Data mart, generating the crosstabulated table in parallel processes. Near lineary performance improvement for the crosstab step.
modified:
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/JDBCCrossTabStore.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml
dhis-2/dhis-services/dhis-service-datamart-default/src/test/java/org/hisp/dhis/datamart/crosstab/CrossTabServiceTest.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java 2011-06-06 05:46:14 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java 2011-10-22 08:41:00 +0000
@@ -31,6 +31,7 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.Future;
import org.hisp.dhis.dataelement.DataElementOperand;
import org.hisp.dhis.datamart.CrossTabDataValue;
@@ -50,6 +51,8 @@
* @return the DataElementOperands with data.
*/
Set<DataElementOperand> getOperandsWithData( Set<DataElementOperand> operands );
+
+ String createCrossTabTable( List<DataElementOperand> operands );
/**
* Creates and populates the crosstab table. Operands without data will be
@@ -60,8 +63,8 @@
* @param organisationUnitIds the collection of OrganisationUnit identifiers.
* @return a List of random keys for each generated crosstab table.
*/
- String populateCrossTabTable( List<DataElementOperand> operands,
- Collection<Integer> periodIds, Collection<Integer> organisationUnitIds );
+ Future<?> populateCrossTabTable( List<DataElementOperand> operands,
+ Collection<Integer> periodIds, Collection<Integer> organisationUnitIds, String key );
/**
* Drops the crosstab table.
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java 2011-06-04 16:56:41 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java 2011-10-22 08:41:00 +0000
@@ -32,9 +32,11 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.Future;
import org.amplecode.quick.BatchHandler;
import org.amplecode.quick.BatchHandlerFactory;
+import org.amplecode.quick.StatementManager;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -44,6 +46,7 @@
import org.hisp.dhis.datamart.crosstab.jdbc.CrossTabStore;
import org.hisp.dhis.datavalue.DataValueService;
import org.hisp.dhis.jdbc.batchhandler.GenericBatchHandler;
+import org.springframework.scheduling.annotation.Async;
/**
* @author Lars Helge Overland
@@ -86,6 +89,13 @@
{
this.dataValueService = dataValueService;
}
+
+ private StatementManager statementManager;
+
+ public void setStatementManager( StatementManager statementManager )
+ {
+ this.statementManager = statementManager;
+ }
// -------------------------------------------------------------------------
// CrossTabService implementation
@@ -96,62 +106,67 @@
return dataValueService.getOperandsWithDataValues( operands );
}
- public String populateCrossTabTable( List<DataElementOperand> operands,
- Collection<Integer> periodIds, Collection<Integer> organisationUnitIds )
+ public String createCrossTabTable( List<DataElementOperand> operands )
{
final String key = RandomStringUtils.randomAlphanumeric( 8 );
- if ( validate( operands, periodIds, organisationUnitIds ) )
+ crossTabStore.dropCrossTabTable( key );
+ crossTabStore.createCrossTabTable( operands, key );
+
+ return key;
+ }
+
+ @Async
+ public Future<?> populateCrossTabTable( List<DataElementOperand> operands,
+ Collection<Integer> periodIds, Collection<Integer> organisationUnitIds, String key )
+ {
+ statementManager.initialise();
+
+ final BatchHandler<Object> batchHandler = batchHandlerFactory.createBatchHandler( GenericBatchHandler.class ).
+ setTableName( CrossTabStore.CROSSTAB_TABLE_PREFIX + key ).init();
+
+ for ( final Integer periodId : periodIds )
{
- crossTabStore.dropCrossTabTable( key );
- crossTabStore.createCrossTabTable( operands, key );
-
- final BatchHandler<Object> batchHandler = batchHandlerFactory.createBatchHandler( GenericBatchHandler.class ).
- setTableName( CrossTabStore.CROSSTAB_TABLE_PREFIX + key ).init();
-
- for ( final Integer periodId : periodIds )
+ for ( final Integer sourceId : organisationUnitIds )
{
- for ( final Integer sourceId : organisationUnitIds )
- {
- final Map<DataElementOperand, String> map = aggregatedDataValueService.getDataValueMap( periodId, sourceId );
-
- final List<String> valueList = new ArrayList<String>( operands.size() + 2 );
-
- valueList.add( String.valueOf( periodId ) );
- valueList.add( String.valueOf( sourceId ) );
-
- boolean hasValues = false;
-
- for ( DataElementOperand operand : operands )
- {
- String value = map.get( operand );
-
- if ( value != null && value.length() > MAX_LENGTH )
- {
- log.warn( "Value ignored, too long: '" + value + "'" );
- value = null;
- }
-
- if ( value != null )
- {
- hasValues = true;
- }
-
- valueList.add( value );
- }
-
- if ( hasValues )
- {
- batchHandler.addObject( valueList );
- }
+ final Map<DataElementOperand, String> map = aggregatedDataValueService.getDataValueMap( periodId, sourceId );
+
+ final List<String> valueList = new ArrayList<String>( operands.size() + 2 );
+
+ valueList.add( String.valueOf( periodId ) );
+ valueList.add( String.valueOf( sourceId ) );
+
+ boolean hasValues = false;
+
+ for ( DataElementOperand operand : operands )
+ {
+ String value = map.get( operand );
+
+ if ( value != null && value.length() > MAX_LENGTH )
+ {
+ log.warn( "Value ignored, too long: '" + value + "'" );
+ value = null;
+ }
+
+ if ( value != null )
+ {
+ hasValues = true;
+ }
+
+ valueList.add( value );
+ }
+
+ if ( hasValues )
+ {
+ batchHandler.addObject( valueList );
}
}
-
- batchHandler.flush();
-
- return key;
}
-
+
+ batchHandler.flush();
+
+ statementManager.destroy();
+
return null;
}
@@ -187,36 +202,4 @@
{
return crossTabStore.getAggregatedDataCacheValue( operands, periodId, sourceId, key );
}
-
- // -------------------------------------------------------------------------
- // Supportive methods
- // -------------------------------------------------------------------------
-
- /**
- * Validates whether the given collections of identifiers are not null and
- * of size greater than 0.
- */
- private boolean validate( Collection<DataElementOperand> operands, Collection<Integer> periodIds,
- Collection<Integer> unitIds )
- {
- if ( operands == null || operands.size() == 0 )
- {
- log.warn( "No operands selected for crosstab table" );
- return false;
- }
-
- if ( periodIds == null || periodIds.size() == 0 )
- {
- log.warn( "No periods selected for crosstab table" );
- return false;
- }
-
- if ( unitIds == null || unitIds.size() == 0 )
- {
- log.warn( "No organisation units selected for crosstab table" );
- return false;
- }
-
- return true;
- }
}
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/JDBCCrossTabStore.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/JDBCCrossTabStore.java 2011-10-19 14:01:02 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/JDBCCrossTabStore.java 2011-10-22 08:41:00 +0000
@@ -173,7 +173,7 @@
try
{
- final Map<DataElementOperand, Double> valueMap = new HashMap<DataElementOperand, Double>();
+ final Map<DataElementOperand, Double> valueMap = new HashMap<DataElementOperand, Double>( operands.size() );
final ResultSet resultSet = holder.getStatement().executeQuery( sql );
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2011-10-08 13:31:38 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2011-10-22 08:41:00 +0000
@@ -223,10 +223,21 @@
state.setMessage( "crosstabulating_data" );
+ Collection<Integer> intersectingPeriodIds = ConversionUtils.getIdentifiers( Period.class, periodService.getIntersectionPeriods( periods ) );
Collection<Integer> childrenIds = organisationUnitService.getOrganisationUnitHierarchy().getChildren( organisationUnitIds );
- Collection<Integer> intersectingPeriodIds = ConversionUtils.getIdentifiers( Period.class, periodService.getIntersectionPeriods( periods ) );
-
- String key = crossTabService.populateCrossTabTable( new ArrayList<DataElementOperand>( allOperands ), intersectingPeriodIds, childrenIds );
+ List<List<Integer>> childrenPages = new PaginatedList<Integer>( childrenIds ).setNumberOfPages( cpuCores ).getPages();
+
+ List<DataElementOperand> crossTabOperands = new ArrayList<DataElementOperand>( allOperands );
+ String key = crossTabService.createCrossTabTable( crossTabOperands );
+
+ List<Future<?>> crossTabFutures = new ArrayList<Future<?>>();
+
+ for ( List<Integer> childrenPage : childrenPages )
+ {
+ crossTabFutures.add( crossTabService.populateCrossTabTable( crossTabOperands, intersectingPeriodIds, childrenPage, key ) );
+ }
+
+ ConcurrentUtils.waitForCompletion( crossTabFutures );
clock.logTime( "Populated crosstab table" );
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml 2011-10-18 20:48:23 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml 2011-10-22 08:41:00 +0000
@@ -53,6 +53,7 @@
<property name="crossTabStore" ref="org.hisp.dhis.datamart.crosstab.jdbc.CrossTabStore" />
<property name="aggregatedDataValueService" ref="org.hisp.dhis.aggregation.AggregatedDataValueService" />
<property name="dataValueService" ref="org.hisp.dhis.datavalue.DataValueService" />
+ <property name="statementManager" ref="statementManager" />
</bean>
<!-- AggregationCache -->
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/test/java/org/hisp/dhis/datamart/crosstab/CrossTabServiceTest.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/test/java/org/hisp/dhis/datamart/crosstab/CrossTabServiceTest.java 2011-06-04 16:56:41 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/test/java/org/hisp/dhis/datamart/crosstab/CrossTabServiceTest.java 2011-10-22 08:41:00 +0000
@@ -189,8 +189,10 @@
@Test
public void testPopulateCrossTabValue()
+ throws Exception
{
- String key = crossTabService.populateCrossTabTable( operands, periodIds, organisationUnitIds );
+ String key = crossTabService.createCrossTabTable( operands );
+ crossTabService.populateCrossTabTable( operands, periodIds, organisationUnitIds, key ).get();
Collection<CrossTabDataValue> values = crossTabService.getCrossTabDataValues( operands, periodIds, organisationUnitIds, key );
