dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #18775
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 7956: Data mart, creating the aggregated data cache after data element data export instead of during th...
------------------------------------------------------------
revno: 7956
committer: Lars Helge Øverland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Wed 2012-08-22 18:13:29 +0200
message:
Data mart, creating the aggregated data cache after data element data export instead of during the export.
modified:
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/DataMartManager.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/JdbcDataMartManager.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DataElementDataMart.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DefaultDataElementDataMart.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/DataMartManager.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/DataMartManager.java 2012-07-26 20:37:31 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/DataMartManager.java 2012-08-22 16:13:29 +0000
@@ -19,6 +19,8 @@
Map<DataElementOperand, String> getDataValueMap( int periodId, int sourceId );
+ Map<DataElementOperand, String> getAggregatedValueMap( int periodId, int organisationUnitId );
+
void createDataValueIndex();
void createIndicatorValueIndex();
@@ -40,6 +42,8 @@
* @param periodIds a collection of Period identifiers.
*/
void deleteAggregatedIndicatorValues( Collection<Integer> periodIds );
+
+ Map<DataElementOperand, String> getAggregatedOrgUnitValueMap( int periodId, int organisationUnitId, int organisationUnitGroupId );
void createOrgUnitDataValueIndex();
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/JdbcDataMartManager.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/JdbcDataMartManager.java 2012-07-26 20:37:31 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/JdbcDataMartManager.java 2012-08-22 16:13:29 +0000
@@ -140,9 +140,46 @@
// -------------------------------------------------------------------------
@Override
+ public Map<DataElementOperand, String> getAggregatedValueMap( int periodId, int organisationUnitId )
+ {
+ final StatementHolder holder = statementManager.getHolder();
+
+ try
+ {
+ final String sql =
+ "SELECT dataelementid, categoryoptioncomboid, value " +
+ "FROM aggregateddatavalue " +
+ "WHERE periodid = " + periodId + " " +
+ "AND organisationunitid = " + organisationUnitId;
+
+ final ResultSet resultSet = holder.getStatement().executeQuery( sql );
+
+ final Map<DataElementOperand, String> map = new HashMap<DataElementOperand, String>();
+
+ while ( resultSet.next() )
+ {
+ final DataElementOperand operand = new DataElementOperand( resultSet.getInt( 1 ), resultSet.getInt( 2 ) );
+
+ map.put( operand, resultSet.getString( 3 ) );
+ }
+
+ return map;
+ }
+ catch ( SQLException ex )
+ {
+ throw new RuntimeException( "Failed to get AggregatedDataValues", ex );
+ }
+ finally
+ {
+ holder.close();
+ }
+ }
+
+ @Override
public void createDataValueIndex()
{
executeSilently( "CREATE INDEX aggregateddatavalue_index ON aggregateddatavalue (dataelementid, categoryoptioncomboid, periodid, organisationunitid)" );
+ executeSilently( "CREATE INDEX aggregateddatavalue_period_index ON aggregateddatavalue (periodid, organisationunitid)" );
}
@Override
@@ -155,6 +192,7 @@
public void dropDataValueIndex()
{
executeSilently( "DROP INDEX aggregateddatavalue_index" );
+ executeSilently( "DROP INDEX aggregateddatavalue_period_index" );
}
@Override
@@ -188,9 +226,47 @@
// -------------------------------------------------------------------------
@Override
+ public Map<DataElementOperand, String> getAggregatedOrgUnitValueMap( int periodId, int organisationUnitId, int organisationUnitGroupId )
+ {
+ final StatementHolder holder = statementManager.getHolder();
+
+ try
+ {
+ final String sql =
+ "SELECT dataelementid, categoryoptioncomboid, value " +
+ "FROM aggregatedorgunitdatavalue " +
+ "WHERE periodid = " + periodId + " " +
+ "AND organisationunitid = " + organisationUnitId + " " +
+ "AND organisationunitgroupid = " + organisationUnitGroupId;
+
+ final ResultSet resultSet = holder.getStatement().executeQuery( sql );
+
+ final Map<DataElementOperand, String> map = new HashMap<DataElementOperand, String>();
+
+ while ( resultSet.next() )
+ {
+ final DataElementOperand operand = new DataElementOperand( resultSet.getInt( 1 ), resultSet.getInt( 2 ) );
+
+ map.put( operand, resultSet.getString( 3 ) );
+ }
+
+ return map;
+ }
+ catch ( SQLException ex )
+ {
+ throw new RuntimeException( "Failed to get AggregatedOrgUnitDataValues", ex );
+ }
+ finally
+ {
+ holder.close();
+ }
+ }
+
+ @Override
public void createOrgUnitDataValueIndex()
{
executeSilently( "CREATE INDEX aggregatedorgunitdatavalue_index ON aggregatedorgunitdatavalue (dataelementid, categoryoptioncomboid, periodid, organisationunitid, organisationunitgroupid)" );
+ executeSilently( "CREATE INDEX aggregatedorgunitdatavalue_period_index ON aggregatedorgunitdatavalue (periodid, organisationunitid, organisationunitgroupid)" );
}
@Override
@@ -203,6 +279,7 @@
public void dropOrgUnitDataValueIndex()
{
executeSilently( "DROP INDEX aggregatedorgunitdatavalue_index" );
+ executeSilently( "DROP INDEX aggregatedorgunitdatavalue_period_index" );
}
@Override
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java 2012-07-25 16:22:03 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java 2012-08-22 16:13:29 +0000
@@ -74,6 +74,9 @@
*/
void createAggregatedDataCache( List<DataElementOperand> operands, String key );
+ Future<?> populateAggregatedDataCache( List<DataElementOperand> operands,
+ Collection<Period> periods, Collection<OrganisationUnit> organisationUnits, String key );
+
/**
* Drops the aggregated data cache table.
* @param key the key used in the table name.
@@ -90,7 +93,10 @@
* @param key the key to use in table name.
*/
void createAggregatedOrgUnitDataCache( List<DataElementOperand> operands, String key );
-
+
+ Future<?> populateAggregatedOrgUnitDataCache( List<DataElementOperand> operands,
+ Collection<Period> periods, Collection<OrganisationUnit> organisationUnits, Collection<OrganisationUnitGroup> organisationUnitGroups, String key );
+
/**
* Drops the aggregated org unit data cache table.
*
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java 2012-07-25 16:22:03 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java 2012-08-22 16:13:29 +0000
@@ -107,7 +107,7 @@
return key;
}
-
+
@Async
public Future<?> populateCrossTabTable( List<DataElementOperand> operands,
Collection<Integer> periodIds, Collection<Integer> organisationUnitIds, String key )
@@ -190,7 +190,55 @@
{
crossTabStore.createAggregatedDataCache( operands, key );
}
-
+
+ @Async
+ public Future<?> populateAggregatedDataCache( List<DataElementOperand> operands,
+ Collection<Period> periods, Collection<OrganisationUnit> organisationUnits, String key )
+ {
+ statementManager.initialise();
+
+ final BatchHandler<Object> batchHandler = batchHandlerFactory.createBatchHandler( GenericBatchHandler.class ).
+ setTableName( CrossTabStore.AGGREGATEDDATA_CACHE_PREFIX + key ).init();
+
+ for ( final Period period : periods )
+ {
+ for ( final OrganisationUnit organisationUnit : organisationUnits )
+ {
+ final Map<DataElementOperand, String> map = dataMartManager.getAggregatedValueMap( period.getId(), organisationUnit.getId() );
+
+ final List<String> valueList = new ArrayList<String>( operands.size() + 2 );
+
+ valueList.add( String.valueOf( period.getId() ) );
+ valueList.add( String.valueOf( organisationUnit.getId() ) );
+
+ boolean hasValues = false;
+
+ for ( DataElementOperand operand : operands )
+ {
+ String value = map.get( operand );
+
+ if ( value != null )
+ {
+ hasValues = true;
+ }
+
+ valueList.add( value );
+ }
+
+ if ( hasValues )
+ {
+ batchHandler.addObject( valueList );
+ }
+ }
+ }
+
+ batchHandler.flush();
+
+ statementManager.destroy();
+
+ return null;
+ }
+
public void dropAggregatedDataCache( String key )
{
crossTabStore.dropAggregatedDataCache( key );
@@ -200,7 +248,59 @@
{
crossTabStore.createAggregatedOrgUnitDataCache( operands, key );
}
-
+
+ @Async
+ public Future<?> populateAggregatedOrgUnitDataCache( List<DataElementOperand> operands,
+ Collection<Period> periods, Collection<OrganisationUnit> organisationUnits, Collection<OrganisationUnitGroup> organisationUnitGroups, String key )
+ {
+ statementManager.initialise();
+
+ final BatchHandler<Object> batchHandler = batchHandlerFactory.createBatchHandler( GenericBatchHandler.class ).
+ setTableName( CrossTabStore.AGGREGATEDORGUNITDATA_CACHE_PREFIX + key ).init();
+
+ for ( final Period period : periods )
+ {
+ for ( final OrganisationUnitGroup group : organisationUnitGroups )
+ {
+ for ( final OrganisationUnit organisationUnit : organisationUnits )
+ {
+ final Map<DataElementOperand, String> map = dataMartManager.getAggregatedOrgUnitValueMap( period.getId(), organisationUnit.getId(), group.getId() );
+
+ final List<String> valueList = new ArrayList<String>( operands.size() + 2 );
+
+ valueList.add( String.valueOf( period.getId() ) );
+ valueList.add( String.valueOf( organisationUnit.getId() ) );
+ valueList.add( String.valueOf( group.getId() ) );
+
+ boolean hasValues = false;
+
+ for ( DataElementOperand operand : operands )
+ {
+ String value = map.get( operand );
+
+ if ( value != null )
+ {
+ hasValues = true;
+ }
+
+ valueList.add( value );
+ }
+
+ if ( hasValues )
+ {
+ batchHandler.addObject( valueList );
+ }
+ }
+ }
+ }
+
+ batchHandler.flush();
+
+ statementManager.destroy();
+
+ return null;
+ }
+
public void dropAggregatedOrgUnitDataCache( String key )
{
crossTabStore.dropAggregatedOrgUnitDataCache( key );
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DataElementDataMart.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DataElementDataMart.java 2011-12-26 10:07:59 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DataElementDataMart.java 2012-08-22 16:13:29 +0000
@@ -33,7 +33,6 @@
import org.amplecode.quick.BatchHandler;
import org.hisp.dhis.aggregation.AggregatedDataValue;
import org.hisp.dhis.dataelement.DataElementOperand;
-import org.hisp.dhis.datamart.DataElementOperandList;
import org.hisp.dhis.organisationunit.OrganisationUnit;
import org.hisp.dhis.organisationunit.OrganisationUnitGroup;
import org.hisp.dhis.organisationunit.OrganisationUnitHierarchy;
@@ -46,5 +45,5 @@
{
Future<?> exportDataValues( Collection<DataElementOperand> operands, Collection<Period> periods,
Collection<OrganisationUnit> organisationUnits, Collection<OrganisationUnitGroup> organisationUnitGroups,
- DataElementOperandList operandList, OrganisationUnitHierarchy hierarchy, Class<? extends BatchHandler<AggregatedDataValue>> clazz, String key );
+ OrganisationUnitHierarchy hierarchy, Class<? extends BatchHandler<AggregatedDataValue>> clazz, String key );
}
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DefaultDataElementDataMart.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DefaultDataElementDataMart.java 2012-02-12 20:32:14 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DefaultDataElementDataMart.java 2012-08-22 16:13:29 +0000
@@ -27,8 +27,6 @@
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
-import static org.hisp.dhis.datamart.crosstab.jdbc.CrossTabStore.AGGREGATEDDATA_CACHE_PREFIX;
-import static org.hisp.dhis.datamart.crosstab.jdbc.CrossTabStore.AGGREGATEDORGUNITDATA_CACHE_PREFIX;
import static org.hisp.dhis.system.util.MathUtils.getRounded;
import java.util.Collection;
@@ -44,11 +42,9 @@
import org.apache.commons.logging.LogFactory;
import org.hisp.dhis.aggregation.AggregatedDataValue;
import org.hisp.dhis.dataelement.DataElementOperand;
-import org.hisp.dhis.datamart.DataElementOperandList;
+import org.hisp.dhis.datamart.DataMartEngine;
import org.hisp.dhis.datamart.aggregation.cache.AggregationCache;
import org.hisp.dhis.datamart.aggregation.dataelement.DataElementAggregator;
-import org.hisp.dhis.datamart.DataMartEngine;
-import org.hisp.dhis.jdbc.batchhandler.GenericBatchHandler;
import org.hisp.dhis.organisationunit.OrganisationUnit;
import org.hisp.dhis.organisationunit.OrganisationUnitGroup;
import org.hisp.dhis.organisationunit.OrganisationUnitHierarchy;
@@ -77,13 +73,6 @@
this.batchHandlerFactory = batchHandlerFactory;
}
- private BatchHandlerFactory inMemoryBatchHandlerFactory;
-
- public void setInMemoryBatchHandlerFactory( BatchHandlerFactory inMemoryBatchHandlerFactory )
- {
- this.inMemoryBatchHandlerFactory = inMemoryBatchHandlerFactory;
- }
-
private AggregationCache aggregationCache;
public void setAggregationCache( AggregationCache aggregationCache )
@@ -140,16 +129,12 @@
@Async
public Future<?> exportDataValues( Collection<DataElementOperand> operands, Collection<Period> periods,
Collection<OrganisationUnit> organisationUnits, Collection<OrganisationUnitGroup> organisationUnitGroups,
- DataElementOperandList operandList, OrganisationUnitHierarchy hierarchy, Class<? extends BatchHandler<AggregatedDataValue>> clazz, String key )
+ OrganisationUnitHierarchy hierarchy, Class<? extends BatchHandler<AggregatedDataValue>> clazz, String key )
{
statementManager.initialise(); // Running in separate thread
final BatchHandler<AggregatedDataValue> batchHandler = batchHandlerFactory.createBatchHandler( clazz ).init();
- final String tableName = organisationUnitGroups != null ? AGGREGATEDORGUNITDATA_CACHE_PREFIX : AGGREGATEDDATA_CACHE_PREFIX;
-
- final BatchHandler<Object> cacheHandler = inMemoryBatchHandlerFactory.createBatchHandler( GenericBatchHandler.class ).setTableName( tableName + key ).init();
-
final Map<DataElementOperand, Double> valueMap = new HashMap<DataElementOperand, Double>();
final AggregatedDataValue aggregatedValue = new AggregatedDataValue();
@@ -168,8 +153,6 @@
{
for ( final OrganisationUnit unit : organisationUnits )
{
- operandList.init( period, unit, group );
-
final int level = aggregationCache.getLevelOfOrganisationUnit( unit.getId() );
final Collection<Integer> orgUnitChildren = hierarchy.getChildren( unit.getId(), group );
@@ -199,15 +182,8 @@
aggregatedValue.setValue( value );
batchHandler.addObject( aggregatedValue );
-
- operandList.addValue( entry.getKey(), value );
}
}
-
- if ( operandList.hasValues() )
- {
- cacheHandler.addObject( operandList.getList() );
- }
}
}
@@ -216,8 +192,6 @@
batchHandler.flush();
- cacheHandler.flush();
-
statementManager.destroy();
aggregationCache.clearCache();
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2012-08-05 20:08:25 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2012-08-22 16:13:29 +0000
@@ -44,7 +44,6 @@
import org.hisp.dhis.dataelement.DataElementCategoryService;
import org.hisp.dhis.dataelement.DataElementOperand;
import org.hisp.dhis.dataelement.DataElementService;
-import org.hisp.dhis.datamart.DataElementOperandList;
import org.hisp.dhis.datamart.DataMartEngine;
import org.hisp.dhis.datamart.DataMartManager;
import org.hisp.dhis.datamart.crosstab.CrossTabService;
@@ -315,22 +314,14 @@
ConcurrentUtils.waitForCompletion( crossTabFutures );
clock.logTime( "Populated crosstab table, " + SystemUtils.getMemoryString() );
+ notifier.notify( id, DATAMART, "Exporting data element data" );
final boolean isIndicators = indicators != null && indicators.size() > 0;
// ---------------------------------------------------------------------
- // 1. Create aggregated data cache
- // ---------------------------------------------------------------------
-
- crossTabService.createAggregatedDataCache( indicatorOperands, key );
-
- clock.logTime( "Created aggregated data cache, number of indicator operands: " + indicatorOperands.size() + ", operands with data: " + allOperands.size() );
- notifier.notify( id, DATAMART, "Exporting data for data element data" );
-
- // ---------------------------------------------------------------------
- // 2. Export data element values
- // ---------------------------------------------------------------------
-
+ // 1. Export data element values
+ // ---------------------------------------------------------------------
+
List<List<OrganisationUnit>> organisationUnitPages = new PaginatedList<OrganisationUnit>( organisationUnits ).setNumberOfPages( cpuCores ).getPages();
if ( allOperands.size() > 0 )
@@ -342,7 +333,7 @@
for ( List<OrganisationUnit> organisationUnitPage : organisationUnitPages )
{
futures.add( dataElementDataMart.exportDataValues( allOperands, periods, organisationUnitPage,
- null, new DataElementOperandList( indicatorOperands ), hierarchy, AggregatedDataValueTempBatchHandler.class, key ) );
+ null, hierarchy, AggregatedDataValueTempBatchHandler.class, key ) );
}
ConcurrentUtils.waitForCompletion( futures );
@@ -352,7 +343,7 @@
notifier.notify( id, DATAMART, "Dropping data element index" );
// ---------------------------------------------------------------------
- // 3. Drop data element index
+ // 2. Drop data element index
// ---------------------------------------------------------------------
dataMartManager.dropDataValueIndex();
@@ -361,7 +352,7 @@
notifier.notify( id, DATAMART, "Deleting existing data element data" );
// ---------------------------------------------------------------------
- // 4. Delete existing aggregated data values
+ // 3. Delete existing aggregated data values
// ---------------------------------------------------------------------
dataMartManager.deleteAggregatedDataValues( periodIds );
@@ -370,7 +361,7 @@
notifier.notify( id, DATAMART, "Copying data element data from temporary table" );
// ---------------------------------------------------------------------
- // 5. Copy aggregated data values from temporary table
+ // 4. Copy aggregated data values from temporary table
// ---------------------------------------------------------------------
dataMartManager.copyAggregatedDataValuesFromTemp();
@@ -379,20 +370,40 @@
notifier.notify( id, DATAMART, "Creating data element index" );
// ---------------------------------------------------------------------
- // 6. Create data element index
+ // 5. Create data element index
// ---------------------------------------------------------------------
dataMartManager.createDataValueIndex();
clock.logTime( "Created data element index" );
- notifier.notify( id, DATAMART, "Exporting data for indicator data" );
-
- // ---------------------------------------------------------------------
- // 7. Export indicator values
- // ---------------------------------------------------------------------
if ( isIndicators )
{
+ // -----------------------------------------------------------------
+ // 6. Create and populate aggregated data cache
+ // -----------------------------------------------------------------
+
+ notifier.notify( id, DATAMART, "Populating aggregated data cache" );
+
+ crossTabService.createAggregatedDataCache( indicatorOperands, key );
+
+ List<Future<?>> aggregatedDataCacheFutures = new ArrayList<Future<?>>();
+
+ for ( List<OrganisationUnit> organisationUnitPage : organisationUnitPages )
+ {
+ aggregatedDataCacheFutures.add( crossTabService.populateAggregatedDataCache(
+ indicatorOperands, periods, organisationUnitPage, key ) );
+ }
+
+ ConcurrentUtils.waitForCompletion( aggregatedDataCacheFutures );
+
+ clock.logTime( "Created aggregated data cache, number of indicator operands: " + indicatorOperands.size() + ", operands with data: " + allOperands.size() );
+ notifier.notify( id, DATAMART, "Exporting indicator data" );
+
+ // -----------------------------------------------------------------
+ // 7. Export indicator values
+ // -----------------------------------------------------------------
+
List<Future<?>> futures = new ArrayList<Future<?>>();
for ( List<OrganisationUnit> organisationUnitPage : organisationUnitPages )
@@ -402,46 +413,47 @@
}
ConcurrentUtils.waitForCompletion( futures );
+
+ clock.logTime( "Exported values for indicators (" + indicators.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString() );
+ notifier.notify( id, DATAMART, "Dropping indicator index" );
+
+ // -----------------------------------------------------------------
+ // 8. Drop aggregated data cache and indicator index
+ // -----------------------------------------------------------------
+
+ crossTabService.dropAggregatedDataCache( key );
+ dataMartManager.dropIndicatorValueIndex();
+
+ clock.logTime( "Dropped indicator index, " + SystemUtils.getMemoryString() );
+ notifier.notify( id, DATAMART, "Deleting existing indicator data" );
+
+ // -----------------------------------------------------------------
+ // 9. Delete existing aggregated indicator values
+ // -----------------------------------------------------------------
+
+ dataMartManager.deleteAggregatedIndicatorValues( periodIds );
+
+ clock.logTime( "Deleted existing indicator data" );
+ notifier.notify( id, DATAMART, "Copying indicator data from temporary table" );
+
+ // -----------------------------------------------------------------
+ // 10. Copy aggregated data values from temporary table
+ // -----------------------------------------------------------------
+
+ dataMartManager.copyAggregatedIndicatorValuesFromTemp();
+
+ clock.logTime( "Copied indicator data from temporary table" );
+ notifier.notify( id, DATAMART, "Creating indicator index" );
+
+ // -----------------------------------------------------------------
+ // 11. Create indicator index
+ // -----------------------------------------------------------------
+
+ dataMartManager.createIndicatorValueIndex();
+
+ clock.logTime( "Created indicator index" );
}
-
- clock.logTime( "Exported values for indicators (" + indicators.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString() );
- notifier.notify( id, DATAMART, "Dropping indicator index" );
-
- // ---------------------------------------------------------------------
- // 8. Drop aggregated data cache and indicator index
- // ---------------------------------------------------------------------
-
- crossTabService.dropAggregatedDataCache( key );
- dataMartManager.dropIndicatorValueIndex();
-
- clock.logTime( "Dropped indicator index, " + SystemUtils.getMemoryString() );
- notifier.notify( id, DATAMART, "Deleting existing indicator data" );
-
- // ---------------------------------------------------------------------
- // 9. Delete existing aggregated indicator values
- // ---------------------------------------------------------------------
-
- dataMartManager.deleteAggregatedIndicatorValues( periodIds );
-
- clock.logTime( "Deleted existing indicator data" );
- notifier.notify( id, DATAMART, "Copying indicator data from temporary table" );
-
- // ---------------------------------------------------------------------
- // 10. Copy aggregated data values from temporary table
- // ---------------------------------------------------------------------
-
- dataMartManager.copyAggregatedIndicatorValuesFromTemp();
-
- clock.logTime( "Copied indicator data from temporary table" );
- notifier.notify( id, DATAMART, "Creating indicator index" );
-
- // ---------------------------------------------------------------------
- // 11. Create indicator index
- // ---------------------------------------------------------------------
-
- dataMartManager.createIndicatorValueIndex();
-
- clock.logTime( "Created indicator index" );
+
clock.logTime( "Aggregated data export done" );
final boolean isGroups = organisationUnitGroups != null && organisationUnitGroups.size() > 0;
@@ -450,19 +462,12 @@
if ( isGroups && groupLevel > 0 )
{
- // -----------------------------------------------------------------
- // 1. Create aggregated data cache
- // -----------------------------------------------------------------
-
- crossTabService.createAggregatedOrgUnitDataCache( indicatorOperands, key );
-
- clock.logTime( "Created aggregated org unit data cache" );
- notifier.notify( id, DATAMART, "Exporting org unit data element data" );
-
// ---------------------------------------------------------------------
- // 2. Export data element values
+ // 1. Export data element values
// ---------------------------------------------------------------------
+ notifier.notify( id, DATAMART, "Exporting data element org unit data" );
+
Collection<OrganisationUnit> groupOrganisationUnits = new HashSet<OrganisationUnit>( organisationUnits );
FilterUtils.filter( groupOrganisationUnits, new OrganisationUnitAboveOrEqualToLevelFilter( groupLevel ) );
@@ -478,7 +483,7 @@
for ( List<OrganisationUnit> organisationUnitPage : organisationUnitPages )
{
futures.add( dataElementDataMart.exportDataValues( allOperands, periods, organisationUnitPage,
- organisationUnitGroups, new DataElementOperandList( indicatorOperands ), hierarchy, AggregatedOrgUnitDataValueTempBatchHandler.class, key ) );
+ organisationUnitGroups, hierarchy, AggregatedOrgUnitDataValueTempBatchHandler.class, key ) );
}
ConcurrentUtils.waitForCompletion( futures );
@@ -488,7 +493,7 @@
notifier.notify( id, DATAMART, "Dropping data element data indexes" );
// -----------------------------------------------------------------
- // 3. Drop data element index
+ // 2. Drop data element index
// -----------------------------------------------------------------
dataMartManager.dropOrgUnitDataValueIndex();
@@ -497,7 +502,7 @@
notifier.notify( id, DATAMART, "Deleting existing org unit data element data" );
// ---------------------------------------------------------------------
- // 4. Delete existing aggregated data values
+ // 3. Delete existing aggregated data values
// ---------------------------------------------------------------------
dataMartManager.deleteAggregatedOrgUnitDataValues( periodIds );
@@ -506,7 +511,7 @@
notifier.notify( id, DATAMART, "Copying org unit data element data" );
// ---------------------------------------------------------------------
- // 5. Copy aggregated org unit data values from temporary table
+ // 4. Copy aggregated org unit data values from temporary table
// ---------------------------------------------------------------------
dataMartManager.copyAggregatedOrgUnitDataValuesFromTemp();
@@ -515,7 +520,7 @@
notifier.notify( id, DATAMART, "Creating org unit data element index" );
// ---------------------------------------------------------------------
- // 6. Create org unit data element index
+ // 5. Create org unit data element index
// ---------------------------------------------------------------------
dataMartManager.createOrgUnitDataValueIndex();
@@ -523,12 +528,33 @@
clock.logTime( "Created org unit data element index" );
notifier.notify( id, DATAMART, "Exporting data for org unit indicator data" );
- // ---------------------------------------------------------------------
- // 7. Export indicator values
- // ---------------------------------------------------------------------
-
if ( isIndicators )
{
+ // -----------------------------------------------------------------
+ // 6. Create aggregated data cache
+ // -----------------------------------------------------------------
+
+ notifier.notify( id, DATAMART, "Populating aggregated data cache" );
+
+ crossTabService.createAggregatedOrgUnitDataCache( indicatorOperands, key );
+
+ List<Future<?>> aggregatedDataCacheFutures = new ArrayList<Future<?>>();
+
+ for ( List<OrganisationUnit> organisationUnitPage : organisationUnitPages )
+ {
+ aggregatedDataCacheFutures.add( crossTabService.populateAggregatedOrgUnitDataCache(
+ indicatorOperands, periods, organisationUnitPage, organisationUnitGroups, key ) );
+ }
+
+ ConcurrentUtils.waitForCompletion( aggregatedDataCacheFutures );
+
+ clock.logTime( "Created aggregated org unit data cache" );
+ notifier.notify( id, DATAMART, "Exporting org unit indicator data" );
+
+ // ---------------------------------------------------------------------
+ // 7. Export indicator values
+ // ---------------------------------------------------------------------
+
List<Future<?>> futures = new ArrayList<Future<?>>();
for ( List<OrganisationUnit> organisationUnitPage : organisationUnitPages )
@@ -538,46 +564,47 @@
}
ConcurrentUtils.waitForCompletion( futures );
+
+ clock.logTime( "Exported values for indicators (" + indicators.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString() );
+ notifier.notify( id, DATAMART, "Dropping org unit indicator index" );
+
+ // ---------------------------------------------------------------------
+ // 8. Drop aggregated data cache and indicator index
+ // ---------------------------------------------------------------------
+
+ crossTabService.dropAggregatedOrgUnitDataCache( key );
+ dataMartManager.dropOrgUnitIndicatorValueIndex();
+
+ clock.logTime( "Dropped org unit indicator index, " + SystemUtils.getMemoryString() );
+ notifier.notify( id, DATAMART, "Deleting existing org unit indicator data" );
+
+ // ---------------------------------------------------------------------
+ // 9. Delete existing aggregated indicator values
+ // ---------------------------------------------------------------------
+
+ dataMartManager.deleteAggregatedOrgUnitIndicatorValues( periodIds );
+
+ clock.logTime( "Deleted existing aggregated org unit indicatorvalues" );
+ notifier.notify( id, DATAMART, "Copying org unit indicator data from temporary table" );
+
+ // ---------------------------------------------------------------------
+ // 10. Copy aggregated org unit indicator values from temporary table
+ // ---------------------------------------------------------------------
+
+ dataMartManager.copyAggregatedOrgUnitIndicatorValuesFromTemp();
+
+ clock.logTime( "Copied org unit indicator data from temporary table" );
+ notifier.notify( id, DATAMART, "Creating org unit indicator indexes" );
+
+ // ---------------------------------------------------------------------
+ // 11. Create org unit indicator index
+ // ---------------------------------------------------------------------
+
+ dataMartManager.createOrgUnitIndicatorValueIndex();
+
+ clock.logTime( "Created org unit indicator index" );
}
- clock.logTime( "Exported values for indicators (" + indicators.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString() );
- notifier.notify( id, DATAMART, "Dropping org unit indicator index" );
-
- // ---------------------------------------------------------------------
- // 8. Drop aggregated data cache and indicator index
- // ---------------------------------------------------------------------
-
- crossTabService.dropAggregatedOrgUnitDataCache( key );
- dataMartManager.dropOrgUnitIndicatorValueIndex();
-
- clock.logTime( "Dropped org unit indicator index, " + SystemUtils.getMemoryString() );
- notifier.notify( id, DATAMART, "Deleting existing org unit indicator data" );
-
- // ---------------------------------------------------------------------
- // 9. Delete existing aggregated indicator values
- // ---------------------------------------------------------------------
-
- dataMartManager.deleteAggregatedOrgUnitIndicatorValues( periodIds );
-
- clock.logTime( "Deleted existing aggregated org unit indicatorvalues" );
- notifier.notify( id, DATAMART, "Copying org unit indicator data from temporary table" );
-
- // ---------------------------------------------------------------------
- // 10. Copy aggregated org unit indicator values from temporary table
- // ---------------------------------------------------------------------
-
- dataMartManager.copyAggregatedOrgUnitIndicatorValuesFromTemp();
-
- clock.logTime( "Copied org unit indicator data from temporary table" );
- notifier.notify( id, DATAMART, "Creating org unit indicator indexes" );
-
- // ---------------------------------------------------------------------
- // 11. Create org unit indicator index
- // ---------------------------------------------------------------------
-
- dataMartManager.createOrgUnitIndicatorValueIndex();
-
- clock.logTime( "Created org unit indicator index" );
clock.logTime( "Aggregated org unit data export done" );
}
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml 2012-07-26 16:50:38 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml 2012-08-22 16:13:29 +0000
@@ -81,7 +81,6 @@
<bean id="org.hisp.dhis.datamart.dataelement.DataElementDataMart" class="org.hisp.dhis.datamart.dataelement.DefaultDataElementDataMart">
<property name="batchHandlerFactory" ref="batchHandlerFactory" />
- <property name="inMemoryBatchHandlerFactory" ref="inMemoryBatchHandlerFactory" />
<property name="aggregationCache" ref="org.hisp.dhis.datamart.aggregation.cache.AggregationCache" />
<property name="sumIntAggregator" ref="org.hisp.dhis.datamart.aggregation.dataelement.SumIntAggregator" />
<property name="averageIntAggregator" ref="org.hisp.dhis.datamart.aggregation.dataelement.AverageIntAggregator" />
