dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #18778
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 7959: Data mart, exporting data elements before indicators and dropping crosstab table half-way, which ...
------------------------------------------------------------
revno: 7959
committer: Lars Helge Øverland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Wed 2012-08-22 19:05:32 +0200
message:
Data mart, exporting data elements before indicators and dropping crosstab table half-way, which saves a lot of memory usage during the process.
modified:
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2012-08-22 16:17:24 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2012-08-22 17:05:32 +0000
@@ -316,155 +316,81 @@
clock.logTime( "Populated crosstab table, " + SystemUtils.getMemoryString() );
notifier.notify( id, DATAMART, "Exporting data element data" );
+ final boolean isDataElements = true;
+
final boolean isIndicators = indicators != null && indicators.size() > 0;
-
- // ---------------------------------------------------------------------
- // 1. Export data element values
- // ---------------------------------------------------------------------
-
- List<List<OrganisationUnit>> organisationUnitPages = new PaginatedList<OrganisationUnit>( organisationUnits ).setNumberOfPages( cpuCores ).getPages();
-
- if ( allOperands.size() > 0 )
- {
- final OrganisationUnitHierarchy hierarchy = organisationUnitService.getOrganisationUnitHierarchy().prepareChildren( organisationUnits );
-
- List<Future<?>> futures = new ArrayList<Future<?>>();
-
- for ( List<OrganisationUnit> organisationUnitPage : organisationUnitPages )
- {
- futures.add( dataElementDataMart.exportDataValues( allOperands, periods, organisationUnitPage,
- null, hierarchy, AggregatedDataValueTempBatchHandler.class, key ) );
- }
-
- ConcurrentUtils.waitForCompletion( futures );
- }
-
- clock.logTime( "Exported values for data element operands (" + allOperands.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString() );
- notifier.notify( id, DATAMART, "Dropping data element index" );
-
- // ---------------------------------------------------------------------
- // 2. Drop data element index
- // ---------------------------------------------------------------------
-
- dataMartManager.dropDataValueIndex();
-
- clock.logTime( "Dropped data element index" );
- notifier.notify( id, DATAMART, "Deleting existing data element data" );
-
- // ---------------------------------------------------------------------
- // 3. Delete existing aggregated data values
- // ---------------------------------------------------------------------
-
- dataMartManager.deleteAggregatedDataValues( periodIds );
-
- clock.logTime( "Deleted existing data element data" );
- notifier.notify( id, DATAMART, "Copying data element data from temporary table" );
-
- // ---------------------------------------------------------------------
- // 4. Copy aggregated data values from temporary table
- // ---------------------------------------------------------------------
-
- dataMartManager.copyAggregatedDataValuesFromTemp();
-
- clock.logTime( "Copied data element data from temporary table" );
- notifier.notify( id, DATAMART, "Creating data element index" );
-
- // ---------------------------------------------------------------------
- // 5. Create data element index
- // ---------------------------------------------------------------------
-
- dataMartManager.createDataValueIndex();
-
- clock.logTime( "Created data element index" );
-
- if ( isIndicators )
- {
- // -----------------------------------------------------------------
- // 6. Create and populate aggregated data cache
- // -----------------------------------------------------------------
-
- notifier.notify( id, DATAMART, "Populating aggregated data cache" );
-
- crossTabService.createAggregatedDataCache( indicatorOperands, key );
-
- List<Future<?>> aggregatedDataCacheFutures = new ArrayList<Future<?>>();
-
- for ( List<OrganisationUnit> organisationUnitPage : organisationUnitPages )
- {
- aggregatedDataCacheFutures.add( crossTabService.populateAggregatedDataCache(
- indicatorOperands, periods, organisationUnitPage, key ) );
- }
-
- ConcurrentUtils.waitForCompletion( aggregatedDataCacheFutures );
-
- clock.logTime( "Created aggregated data cache, number of indicator operands: " + indicatorOperands.size() + ", operands with data: " + allOperands.size() );
- notifier.notify( id, DATAMART, "Exporting indicator data" );
-
- // -----------------------------------------------------------------
- // 7. Export indicator values
- // -----------------------------------------------------------------
-
- List<Future<?>> futures = new ArrayList<Future<?>>();
-
- for ( List<OrganisationUnit> organisationUnitPage : organisationUnitPages )
- {
- futures.add( indicatorDataMart.exportIndicatorValues( indicators, periods, organisationUnitPage,
- null, indicatorOperands, AggregatedIndicatorValueTempBatchHandler.class, key ) );
- }
-
- ConcurrentUtils.waitForCompletion( futures );
-
- clock.logTime( "Exported values for indicators (" + indicators.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString() );
- notifier.notify( id, DATAMART, "Dropping indicator index" );
-
- // -----------------------------------------------------------------
- // 8. Drop aggregated data cache and indicator index
- // -----------------------------------------------------------------
-
- crossTabService.dropAggregatedDataCache( key );
- dataMartManager.dropIndicatorValueIndex();
-
- clock.logTime( "Dropped indicator index, " + SystemUtils.getMemoryString() );
- notifier.notify( id, DATAMART, "Deleting existing indicator data" );
-
- // -----------------------------------------------------------------
- // 9. Delete existing aggregated indicator values
- // -----------------------------------------------------------------
-
- dataMartManager.deleteAggregatedIndicatorValues( periodIds );
-
- clock.logTime( "Deleted existing indicator data" );
- notifier.notify( id, DATAMART, "Copying indicator data from temporary table" );
-
- // -----------------------------------------------------------------
- // 10. Copy aggregated data values from temporary table
- // -----------------------------------------------------------------
-
- dataMartManager.copyAggregatedIndicatorValuesFromTemp();
-
- clock.logTime( "Copied indicator data from temporary table" );
- notifier.notify( id, DATAMART, "Creating indicator index" );
-
- // -----------------------------------------------------------------
- // 11. Create indicator index
- // -----------------------------------------------------------------
-
- dataMartManager.createIndicatorValueIndex();
-
- clock.logTime( "Created indicator index" );
- }
-
- clock.logTime( "Aggregated data export done" );
final int groupLevel = (Integer) systemSettingManager.getSystemSetting( KEY_ORGUNITGROUPSET_AGG_LEVEL, DEFAULT_ORGUNITGROUPSET_AGG_LEVEL );
final boolean isGroups = organisationUnitGroups != null && organisationUnitGroups.size() > 0 && groupLevel > 0;
-
- if ( isGroups )
- {
- // ---------------------------------------------------------------------
- // 1. Export data element values
- // ---------------------------------------------------------------------
+
+ List<List<OrganisationUnit>> organisationUnitPages = new PaginatedList<OrganisationUnit>( organisationUnits ).setNumberOfPages( cpuCores ).getPages();
+
+ if ( isDataElements )
+ {
+ // -----------------------------------------------------------------
+ // 1. Export data element values
+ // -----------------------------------------------------------------
+
+ if ( allOperands.size() > 0 )
+ {
+ final OrganisationUnitHierarchy hierarchy = organisationUnitService.getOrganisationUnitHierarchy().prepareChildren( organisationUnits );
+
+ List<Future<?>> futures = new ArrayList<Future<?>>();
+
+ for ( List<OrganisationUnit> organisationUnitPage : organisationUnitPages )
+ {
+ futures.add( dataElementDataMart.exportDataValues( allOperands, periods, organisationUnitPage,
+ null, hierarchy, AggregatedDataValueTempBatchHandler.class, key ) );
+ }
+
+ ConcurrentUtils.waitForCompletion( futures );
+ }
+
+ clock.logTime( "Exported values for data element operands (" + allOperands.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString() );
+ notifier.notify( id, DATAMART, "Dropping data element index" );
+
+ // -----------------------------------------------------------------
+ // 2. Drop data element index
+ // -----------------------------------------------------------------
+
+ dataMartManager.dropDataValueIndex();
+
+ clock.logTime( "Dropped data element index" );
+ notifier.notify( id, DATAMART, "Deleting existing data element data" );
+
+ // -----------------------------------------------------------------
+ // 3. Delete existing aggregated data values
+ // -----------------------------------------------------------------
+
+ dataMartManager.deleteAggregatedDataValues( periodIds );
+
+ clock.logTime( "Deleted existing data element data" );
+ notifier.notify( id, DATAMART, "Copying data element data from temporary table" );
+
+ // -----------------------------------------------------------------
+ // 4. Copy aggregated data values from temporary table
+ // -----------------------------------------------------------------
+
+ dataMartManager.copyAggregatedDataValuesFromTemp();
+
+ clock.logTime( "Copied data element data from temporary table" );
+ notifier.notify( id, DATAMART, "Creating data element index" );
+
+ // -----------------------------------------------------------------
+ // 5. Create data element index
+ // -----------------------------------------------------------------
+
+ dataMartManager.createDataValueIndex();
+
+ clock.logTime( "Created data element index" );
+ }
+
+ if ( isGroups && isDataElements )
+ {
+ // -----------------------------------------------------------------
+ // 1. Export data element values
+ // -----------------------------------------------------------------
notifier.notify( id, DATAMART, "Exporting data element org unit data" );
@@ -501,118 +427,194 @@
clock.logTime( "Dropped org unit data element index" );
notifier.notify( id, DATAMART, "Deleting existing org unit data element data" );
- // ---------------------------------------------------------------------
+ // -----------------------------------------------------------------
// 3. Delete existing aggregated data values
- // ---------------------------------------------------------------------
+ // -----------------------------------------------------------------
dataMartManager.deleteAggregatedOrgUnitDataValues( periodIds );
clock.logTime( "Deleted existing aggregated org unit datavalues" );
notifier.notify( id, DATAMART, "Copying org unit data element data" );
- // ---------------------------------------------------------------------
+ // -----------------------------------------------------------------
// 4. Copy aggregated org unit data values from temporary table
- // ---------------------------------------------------------------------
+ // -----------------------------------------------------------------
dataMartManager.copyAggregatedOrgUnitDataValuesFromTemp();
clock.logTime( "Copied org unit data element data from temporary table" );
notifier.notify( id, DATAMART, "Creating org unit data element index" );
- // ---------------------------------------------------------------------
+ // -----------------------------------------------------------------
// 5. Create org unit data element index
- // ---------------------------------------------------------------------
+ // -----------------------------------------------------------------
dataMartManager.createOrgUnitDataValueIndex();
clock.logTime( "Created org unit data element index" );
notifier.notify( id, DATAMART, "Exporting data for org unit indicator data" );
-
- if ( isIndicators )
- {
- // -----------------------------------------------------------------
- // 6. Create aggregated data cache
- // -----------------------------------------------------------------
-
- notifier.notify( id, DATAMART, "Populating aggregated data cache" );
-
- crossTabService.createAggregatedOrgUnitDataCache( indicatorOperands, key );
-
- List<Future<?>> aggregatedDataCacheFutures = new ArrayList<Future<?>>();
-
- for ( List<OrganisationUnit> organisationUnitPage : organisationUnitPages )
- {
- aggregatedDataCacheFutures.add( crossTabService.populateAggregatedOrgUnitDataCache(
- indicatorOperands, periods, organisationUnitPage, organisationUnitGroups, key ) );
- }
-
- ConcurrentUtils.waitForCompletion( aggregatedDataCacheFutures );
-
- clock.logTime( "Created aggregated org unit data cache" );
- notifier.notify( id, DATAMART, "Exporting org unit indicator data" );
-
- // ---------------------------------------------------------------------
- // 7. Export indicator values
- // ---------------------------------------------------------------------
-
- List<Future<?>> futures = new ArrayList<Future<?>>();
-
- for ( List<OrganisationUnit> organisationUnitPage : organisationUnitPages )
- {
- futures.add( indicatorDataMart.exportIndicatorValues( indicators, periods, organisationUnitPage,
- organisationUnitGroups, indicatorOperands, AggregatedOrgUnitIndicatorValueTempBatchHandler.class, key ) );
- }
-
- ConcurrentUtils.waitForCompletion( futures );
-
- clock.logTime( "Exported values for indicators (" + indicators.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString() );
- notifier.notify( id, DATAMART, "Dropping org unit indicator index" );
-
- // ---------------------------------------------------------------------
- // 8. Drop aggregated data cache and indicator index
- // ---------------------------------------------------------------------
-
- crossTabService.dropAggregatedOrgUnitDataCache( key );
- dataMartManager.dropOrgUnitIndicatorValueIndex();
-
- clock.logTime( "Dropped org unit indicator index, " + SystemUtils.getMemoryString() );
- notifier.notify( id, DATAMART, "Deleting existing org unit indicator data" );
-
- // ---------------------------------------------------------------------
- // 9. Delete existing aggregated indicator values
- // ---------------------------------------------------------------------
-
- dataMartManager.deleteAggregatedOrgUnitIndicatorValues( periodIds );
-
- clock.logTime( "Deleted existing aggregated org unit indicatorvalues" );
- notifier.notify( id, DATAMART, "Copying org unit indicator data from temporary table" );
-
- // ---------------------------------------------------------------------
- // 10. Copy aggregated org unit indicator values from temporary table
- // ---------------------------------------------------------------------
-
- dataMartManager.copyAggregatedOrgUnitIndicatorValuesFromTemp();
-
- clock.logTime( "Copied org unit indicator data from temporary table" );
- notifier.notify( id, DATAMART, "Creating org unit indicator indexes" );
-
- // ---------------------------------------------------------------------
- // 11. Create org unit indicator index
- // ---------------------------------------------------------------------
-
- dataMartManager.createOrgUnitIndicatorValueIndex();
-
- clock.logTime( "Created org unit indicator index" );
- }
-
- clock.logTime( "Aggregated org unit data export done" );
}
- // ---------------------------------------------------------------------
- // Drop crosstab and temporary tables
- // ---------------------------------------------------------------------
-
crossTabService.dropCrossTabTable( key );
+
+ if ( isIndicators )
+ {
+ // -----------------------------------------------------------------
+ // 1. Create and populate aggregated data cache
+ // -----------------------------------------------------------------
+
+ notifier.notify( id, DATAMART, "Populating aggregated data cache" );
+
+ crossTabService.createAggregatedDataCache( indicatorOperands, key );
+
+ List<Future<?>> aggregatedDataCacheFutures = new ArrayList<Future<?>>();
+
+ for ( List<OrganisationUnit> organisationUnitPage : organisationUnitPages )
+ {
+ aggregatedDataCacheFutures.add( crossTabService.populateAggregatedDataCache(
+ indicatorOperands, periods, organisationUnitPage, key ) );
+ }
+
+ ConcurrentUtils.waitForCompletion( aggregatedDataCacheFutures );
+
+ clock.logTime( "Created aggregated data cache, number of indicator operands: " + indicatorOperands.size() + ", operands with data: " + allOperands.size() );
+ notifier.notify( id, DATAMART, "Exporting indicator data" );
+
+ // -----------------------------------------------------------------
+ // 2. Export indicator values
+ // -----------------------------------------------------------------
+
+ List<Future<?>> futures = new ArrayList<Future<?>>();
+
+ for ( List<OrganisationUnit> organisationUnitPage : organisationUnitPages )
+ {
+ futures.add( indicatorDataMart.exportIndicatorValues( indicators, periods, organisationUnitPage,
+ null, indicatorOperands, AggregatedIndicatorValueTempBatchHandler.class, key ) );
+ }
+
+ ConcurrentUtils.waitForCompletion( futures );
+
+ clock.logTime( "Exported values for indicators (" + indicators.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString() );
+ notifier.notify( id, DATAMART, "Dropping indicator index" );
+
+ // -----------------------------------------------------------------
+ // 3. Drop aggregated data cache and indicator index
+ // -----------------------------------------------------------------
+
+ crossTabService.dropAggregatedDataCache( key );
+ dataMartManager.dropIndicatorValueIndex();
+
+ clock.logTime( "Dropped indicator index, " + SystemUtils.getMemoryString() );
+ notifier.notify( id, DATAMART, "Deleting existing indicator data" );
+
+ // -----------------------------------------------------------------
+ // 4. Delete existing aggregated indicator values
+ // -----------------------------------------------------------------
+
+ dataMartManager.deleteAggregatedIndicatorValues( periodIds );
+
+ clock.logTime( "Deleted existing indicator data" );
+ notifier.notify( id, DATAMART, "Copying indicator data from temporary table" );
+
+ // -----------------------------------------------------------------
+ // 5. Copy aggregated data values from temporary table
+ // -----------------------------------------------------------------
+
+ dataMartManager.copyAggregatedIndicatorValuesFromTemp();
+
+ clock.logTime( "Copied indicator data from temporary table" );
+ notifier.notify( id, DATAMART, "Creating indicator index" );
+
+ // -----------------------------------------------------------------
+ // 6. Create indicator index
+ // -----------------------------------------------------------------
+
+ dataMartManager.createIndicatorValueIndex();
+
+ clock.logTime( "Created indicator index" );
+ }
+
+ if ( isGroups && isIndicators )
+ {
+ // -----------------------------------------------------------------
+ // 1. Create aggregated data cache
+ // -----------------------------------------------------------------
+
+ notifier.notify( id, DATAMART, "Populating aggregated data cache" );
+
+ crossTabService.createAggregatedOrgUnitDataCache( indicatorOperands, key );
+
+ List<Future<?>> aggregatedDataCacheFutures = new ArrayList<Future<?>>();
+
+ for ( List<OrganisationUnit> organisationUnitPage : organisationUnitPages )
+ {
+ aggregatedDataCacheFutures.add( crossTabService.populateAggregatedOrgUnitDataCache(
+ indicatorOperands, periods, organisationUnitPage, organisationUnitGroups, key ) );
+ }
+
+ ConcurrentUtils.waitForCompletion( aggregatedDataCacheFutures );
+
+ clock.logTime( "Created aggregated org unit data cache" );
+ notifier.notify( id, DATAMART, "Exporting org unit indicator data" );
+
+ // -----------------------------------------------------------------
+ // 2. Export indicator values
+ // -----------------------------------------------------------------
+
+ List<Future<?>> futures = new ArrayList<Future<?>>();
+
+ for ( List<OrganisationUnit> organisationUnitPage : organisationUnitPages )
+ {
+ futures.add( indicatorDataMart.exportIndicatorValues( indicators, periods, organisationUnitPage,
+ organisationUnitGroups, indicatorOperands, AggregatedOrgUnitIndicatorValueTempBatchHandler.class, key ) );
+ }
+
+ ConcurrentUtils.waitForCompletion( futures );
+
+ clock.logTime( "Exported values for indicators (" + indicators.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString() );
+ notifier.notify( id, DATAMART, "Dropping org unit indicator index" );
+
+ // -----------------------------------------------------------------
+ // 3. Drop aggregated data cache and indicator index
+ // -----------------------------------------------------------------
+
+ crossTabService.dropAggregatedOrgUnitDataCache( key );
+ dataMartManager.dropOrgUnitIndicatorValueIndex();
+
+ clock.logTime( "Dropped org unit indicator index, " + SystemUtils.getMemoryString() );
+ notifier.notify( id, DATAMART, "Deleting existing org unit indicator data" );
+
+ // -----------------------------------------------------------------
+ // 4. Delete existing aggregated indicator values
+ // -----------------------------------------------------------------
+
+ dataMartManager.deleteAggregatedOrgUnitIndicatorValues( periodIds );
+
+ clock.logTime( "Deleted existing aggregated org unit indicatorvalues" );
+ notifier.notify( id, DATAMART, "Copying org unit indicator data from temporary table" );
+
+ // -----------------------------------------------------------------
+ // 5. Copy aggregated org unit indicator values from temp table
+ // -----------------------------------------------------------------
+
+ dataMartManager.copyAggregatedOrgUnitIndicatorValuesFromTemp();
+
+ clock.logTime( "Copied org unit indicator data from temporary table" );
+ notifier.notify( id, DATAMART, "Creating org unit indicator indexes" );
+
+ // -----------------------------------------------------------------
+ // 6. Create org unit indicator index
+ // -----------------------------------------------------------------
+
+ dataMartManager.createOrgUnitIndicatorValueIndex();
+
+ clock.logTime( "Created org unit indicator index" );
+ }
+
+ // ---------------------------------------------------------------------
+ // Drop temporary tables
+ // ---------------------------------------------------------------------
+
dataMartManager.dropTempAggregatedTables();
clock.logTime( "Dropped crosstab table" );
