dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #06349
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 2002: Performance improvement in datamart when filtering data elements
------------------------------------------------------------
revno: 2002
committer: Lars <larshelg@larshelg-laptop>
branch nick: trunk
timestamp: Thu 2010-06-17 23:58:12 +0200
message:
Performance improvement in datamart when filtering data elements
modified:
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DefaultDataElementDataMart.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DefaultDataElementDataMart.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DefaultDataElementDataMart.java 2010-06-17 21:08:46 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DefaultDataElementDataMart.java 2010-06-17 21:58:12 +0000
@@ -125,29 +125,32 @@
{
final Map<DataElementOperand, Integer> currentOperandIndexMap = dataElementAggregator.getOperandIndexMap( operands, period.getPeriodType(), operandIndexMap );
- for ( final OrganisationUnit unit : organisationUnits )
+ if ( currentOperandIndexMap != null && currentOperandIndexMap.size() > 0 )
{
- final int level = aggregationCache.getLevelOfOrganisationUnit( unit.getId() );
-
- final Map<DataElementOperand, Double> valueMap = dataElementAggregator.getAggregatedValues( currentOperandIndexMap, period, unit, level, hierarchy );
-
- final PeriodType periodType = period.getPeriodType();
-
- for ( Entry<DataElementOperand, Double> entry : valueMap.entrySet() )
+ for ( final OrganisationUnit unit : organisationUnits )
{
- value.clear();
-
- value.setDataElementId( entry.getKey().getDataElementId() );
- value.setCategoryOptionComboId( entry.getKey().getOptionComboId() );
- value.setPeriodId( period.getId() );
- value.setPeriodTypeId( periodType.getId() );
- value.setOrganisationUnitId( unit.getId() );
- value.setLevel( level );
- value.setValue( getRounded( entry.getValue(), DECIMALS ) );
-
- batchHandler.addObject( value );
-
- count++;
+ final int level = aggregationCache.getLevelOfOrganisationUnit( unit.getId() );
+
+ final Map<DataElementOperand, Double> valueMap = dataElementAggregator.getAggregatedValues( currentOperandIndexMap, period, unit, level, hierarchy );
+
+ final PeriodType periodType = period.getPeriodType();
+
+ for ( Entry<DataElementOperand, Double> entry : valueMap.entrySet() )
+ {
+ value.clear();
+
+ value.setDataElementId( entry.getKey().getDataElementId() );
+ value.setCategoryOptionComboId( entry.getKey().getOptionComboId() );
+ value.setPeriodId( period.getId() );
+ value.setPeriodTypeId( periodType.getId() );
+ value.setOrganisationUnitId( unit.getId() );
+ value.setLevel( level );
+ value.setValue( getRounded( entry.getValue(), DECIMALS ) );
+
+ batchHandler.addObject( value );
+
+ count++;
+ }
}
}
}
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2010-06-17 21:08:46 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2010-06-17 21:58:12 +0000
@@ -29,6 +29,8 @@
import static org.hisp.dhis.dataelement.DataElement.VALUE_TYPE_BOOL;
import static org.hisp.dhis.dataelement.DataElement.VALUE_TYPE_INT;
+import static org.hisp.dhis.dataelement.DataElement.AGGREGATION_OPERATOR_SUM;
+import static org.hisp.dhis.dataelement.DataElement.AGGREGATION_OPERATOR_AVERAGE;
import static org.hisp.dhis.datamart.util.ParserUtil.getDataElementIdsInExpression;
import java.util.ArrayList;
@@ -236,10 +238,12 @@
final Collection<DataElementOperand> dataElementInCalculatedDataElementOperands = categoryService.getOperandsByIds( dataElementInCalculatedDataElementIds );
final Collection<DataElementOperand> nonCalculatedDataElementOperands = categoryService.getOperandsByIds( nonCalculatedDataElementIds );
- final Collection<DataElementOperand> intNonCalculatedDataElementOperands = filterOperands( nonCalculatedDataElementOperands, VALUE_TYPE_INT );
- final Collection<DataElementOperand> boolNonCalculatedDataElementOperands = filterOperands( nonCalculatedDataElementOperands, VALUE_TYPE_BOOL );
+ final Collection<DataElementOperand> sumIntDataElementOperands = filterOperands( nonCalculatedDataElementOperands, VALUE_TYPE_INT, AGGREGATION_OPERATOR_SUM );
+ final Collection<DataElementOperand> averageIntDataElementOperands = filterOperands( nonCalculatedDataElementOperands, VALUE_TYPE_INT, AGGREGATION_OPERATOR_AVERAGE );
+ final Collection<DataElementOperand> sumBoolDataElementOperands = filterOperands( nonCalculatedDataElementOperands, VALUE_TYPE_BOOL, AGGREGATION_OPERATOR_SUM );
+ final Collection<DataElementOperand> averageBoolDataElementOperands = filterOperands( nonCalculatedDataElementOperands, VALUE_TYPE_BOOL, AGGREGATION_OPERATOR_AVERAGE );
- log.info( "Filtered data elements" );
+ log.info( "Filtered data elements: " + TimeUtils.getHMS() );
// ---------------------------------------------------------------------
// Create and trim crosstabtable
@@ -276,39 +280,39 @@
state.setMessage( "exporting_data_for_data_elements" );
- if ( intNonCalculatedDataElementOperands.size() > 0 )
- {
- count += dataElementDataMart.exportDataValues( intNonCalculatedDataElementOperands, periodIds, organisationUnitIds, sumIntAggregator );
-
- log.info( "Exported values for data elements with sum aggregation operator of type number: " + TimeUtils.getHMS() );
- }
-
- if ( intNonCalculatedDataElementOperands.size() > 0 )
- {
- count += dataElementDataMart.exportDataValues( intNonCalculatedDataElementOperands, periodIds, organisationUnitIds, averageIntAggregator );
-
- log.info( "Exported values for data elements with average aggregation operator of type number: " + TimeUtils.getHMS() );
- }
-
- if ( intNonCalculatedDataElementOperands.size() > 0 )
- {
- count += dataElementDataMart.exportDataValues( intNonCalculatedDataElementOperands, periodIds, organisationUnitIds, averageIntSingleValueAggregator );
-
- log.info( "Exported values for data elements with average aggregation operator with single value of type number: " + TimeUtils.getHMS() );
- }
-
- if ( boolNonCalculatedDataElementOperands.size() > 0 )
- {
- count += dataElementDataMart.exportDataValues( boolNonCalculatedDataElementOperands, periodIds, organisationUnitIds, sumBoolAggregator );
-
- log.info( "Exported values for data elements with sum aggregation operator of type yes/no: " + TimeUtils.getHMS() );
- }
-
- if ( boolNonCalculatedDataElementOperands.size() > 0 )
- {
- count += dataElementDataMart.exportDataValues( boolNonCalculatedDataElementOperands, periodIds, organisationUnitIds, averageBoolAggregator );
-
- log.info( "Exported values for data elements with average aggregation operator of type yes/no: " + TimeUtils.getHMS() );
+ if ( sumIntDataElementOperands.size() > 0 )
+ {
+ count += dataElementDataMart.exportDataValues( sumIntDataElementOperands, periodIds, organisationUnitIds, sumIntAggregator );
+
+ log.info( "Exported values for data element operands with sum aggregation operator of type number (" + sumIntDataElementOperands.size() + "): " + TimeUtils.getHMS() );
+ }
+
+ if ( averageIntDataElementOperands.size() > 0 )
+ {
+ count += dataElementDataMart.exportDataValues( averageIntDataElementOperands, periodIds, organisationUnitIds, averageIntAggregator );
+
+ log.info( "Exported values for data element operands with average aggregation operator of type number (" + averageIntDataElementOperands.size() + "): " + TimeUtils.getHMS() );
+ }
+
+ if ( averageIntDataElementOperands.size() > 0 )
+ {
+ count += dataElementDataMart.exportDataValues( averageIntDataElementOperands, periodIds, organisationUnitIds, averageIntSingleValueAggregator );
+
+ log.info( "Exported values for data element operands with average aggregation operator with single value of type number (" + averageIntDataElementOperands.size() + "): " + TimeUtils.getHMS() );
+ }
+
+ if ( sumBoolDataElementOperands.size() > 0 )
+ {
+ count += dataElementDataMart.exportDataValues( sumBoolDataElementOperands, periodIds, organisationUnitIds, sumBoolAggregator );
+
+ log.info( "Exported values for data element operands with sum aggregation operator of type yes/no (" + sumBoolDataElementOperands.size() + "): " + TimeUtils.getHMS() );
+ }
+
+ if ( averageBoolDataElementOperands.size() > 0 )
+ {
+ count += dataElementDataMart.exportDataValues( averageBoolDataElementOperands, periodIds, organisationUnitIds, averageBoolAggregator );
+
+ log.info( "Exported values for data element operands with average aggregation operator of type yes/no (" + averageBoolDataElementOperands.size() + "): " + TimeUtils.getHMS() );
}
state.setMessage( "exporting_data_for_indicators" );
@@ -321,7 +325,7 @@
{
count += indicatorDataMart.exportIndicatorValues( indicatorIds, periodIds, organisationUnitIds, dataElementInIndicatorOperands );
- log.info( "Exported values for indicator: " + TimeUtils.getHMS() );
+ log.info( "Exported values for indicators (" + indicatorIds.size() + "): " + TimeUtils.getHMS() );
}
state.setMessage( "exporting_data_for_calculated_data_elements" );
@@ -436,13 +440,13 @@
/**
* Filters the data element operands based on the value type.
*/
- private Collection<DataElementOperand> filterOperands( Collection<DataElementOperand> operands, String valueType )
+ private Collection<DataElementOperand> filterOperands( Collection<DataElementOperand> operands, String valueType, String aggregationOperator )
{
final Collection<DataElementOperand> filtered = new ArrayList<DataElementOperand>();
for ( DataElementOperand operand : operands )
{
- if ( operand.getValueType().equals( valueType ) )
+ if ( operand.getValueType().equals( valueType ) && operand.getAggregationOperator().equals( aggregationOperator ) )
{
filtered.add( operand );
}
