dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #06318
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 1988: Reduced process time for indicator data mart with 50 %. I discovered that aggregation of data ele...
------------------------------------------------------------
revno: 1988
committer: Lars <larshelg@larshelg-laptop>
branch nick: trunk
timestamp: Mon 2010-06-14 18:27:39 +0200
message:
Reduced process time for indicator data mart with 50 %. I discovered that aggregation of data elements with the average aggregation operator was taking a lot longer time than aggregation of sum data elements, since we need to make one query for each organisation unit to determine the average (as opposed to sum, where values for all organisation units can be summarized directly). Still, when the aggregation period periodicity is shorter than the raw data value periodicity there is no need to perform the one-query-per-orgunit since there can only be one value. Those situations can hence be treated similar to sum data elements and improve performance. Will do something similar for data element datamart.
modified:
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageBoolAggregator.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntAggregator.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumBoolAggregator.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumIntAggregator.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/indicator/DefaultIndicatorDataMart.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageBoolAggregator.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageBoolAggregator.java 2010-06-08 19:47:40 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageBoolAggregator.java 2010-06-14 16:27:39 +0000
@@ -76,6 +76,11 @@
public Map<DataElementOperand, Double> getAggregatedValues( final Map<DataElementOperand, Integer> operandIndexMap,
final Period period, final OrganisationUnit unit, int unitLevel, final OrganisationUnitHierarchy hierarchy )
{
+ if ( operandIndexMap == null || operandIndexMap.size() == 0 )
+ {
+ return new HashMap<DataElementOperand, Double>();
+ }
+
final Collection<CrossTabDataValue> crossTabValues =
getCrossTabDataValues( operandIndexMap, period.getStartDate(), period.getEndDate(), unit.getId(), hierarchy );
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntAggregator.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntAggregator.java 2010-06-08 19:47:40 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntAggregator.java 2010-06-14 16:27:39 +0000
@@ -80,6 +80,11 @@
public Map<DataElementOperand, Double> getAggregatedValues( final Map<DataElementOperand, Integer> operandIndexMap,
final Period period, final OrganisationUnit unit, int unitLevel, OrganisationUnitHierarchy hierarchy )
{
+ if ( operandIndexMap == null || operandIndexMap.size() == 0 )
+ {
+ return new HashMap<DataElementOperand, Double>();
+ }
+
final Collection<Integer> unitIds = hierarchy.getChildren( unit.getId() );
final Map<DataElementOperand, Double> values = new HashMap<DataElementOperand, Double>(); // <Operand, total value>
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumBoolAggregator.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumBoolAggregator.java 2010-06-08 19:47:40 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumBoolAggregator.java 2010-06-14 16:27:39 +0000
@@ -77,6 +77,11 @@
public Map<DataElementOperand, Double> getAggregatedValues( final Map<DataElementOperand, Integer> operandIndexMap,
final Period period, final OrganisationUnit unit, int unitLevel, OrganisationUnitHierarchy hierarchy )
{
+ if ( operandIndexMap == null || operandIndexMap.size() == 0 )
+ {
+ return new HashMap<DataElementOperand, Double>();
+ }
+
final Collection<CrossTabDataValue> crossTabValues =
getCrossTabDataValues( operandIndexMap, period.getStartDate(), period.getEndDate(), unit.getId(), hierarchy );
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumIntAggregator.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumIntAggregator.java 2010-06-08 19:47:40 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumIntAggregator.java 2010-06-14 16:27:39 +0000
@@ -80,6 +80,11 @@
public Map<DataElementOperand, Double> getAggregatedValues( final Map<DataElementOperand, Integer> operandIndexMap,
final Period period, final OrganisationUnit unit, int unitLevel, OrganisationUnitHierarchy hierarchy )
{
+ if ( operandIndexMap == null || operandIndexMap.size() == 0 )
+ {
+ return new HashMap<DataElementOperand, Double>();
+ }
+
final Collection<CrossTabDataValue> crossTabValues =
getCrossTabDataValues( operandIndexMap, period.getStartDate(), period.getEndDate(), unit.getId(), hierarchy );
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/indicator/DefaultIndicatorDataMart.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/indicator/DefaultIndicatorDataMart.java 2010-06-08 19:47:40 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/indicator/DefaultIndicatorDataMart.java 2010-06-14 16:27:39 +0000
@@ -27,13 +27,14 @@
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
+import static org.hisp.dhis.dataelement.DataElement.AGGREGATION_OPERATOR_AVERAGE;
+import static org.hisp.dhis.dataelement.DataElement.AGGREGATION_OPERATOR_SUM;
import static org.hisp.dhis.datamart.util.ParserUtil.generateExpression;
import static org.hisp.dhis.options.SystemSettingManager.KEY_OMIT_INDICATORS_ZERO_NUMERATOR_DATAMART;
import static org.hisp.dhis.system.util.DateUtils.DAYS_IN_YEAR;
import static org.hisp.dhis.system.util.MathUtils.calculateExpression;
import static org.hisp.dhis.system.util.MathUtils.getRounded;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
@@ -43,7 +44,6 @@
import org.hisp.dhis.aggregation.AggregatedIndicatorValue;
import org.hisp.dhis.dataelement.DataElement;
import org.hisp.dhis.dataelement.DataElementOperand;
-import org.hisp.dhis.dataelement.DataElementService;
import org.hisp.dhis.datamart.aggregation.cache.AggregationCache;
import org.hisp.dhis.datamart.aggregation.dataelement.DataElementAggregator;
import org.hisp.dhis.datamart.crosstab.CrossTabService;
@@ -117,13 +117,6 @@
{
this.crossTabService = crossTabService;
}
-
- private DataElementService dataElementService;
-
- public void setDataElementService( DataElementService dataElementService )
- {
- this.dataElementService = dataElementService;
- }
private AggregationCache aggregationCache;
@@ -149,11 +142,7 @@
public int exportIndicatorValues( final Collection<Integer> indicatorIds, final Collection<Integer> periodIds,
final Collection<Integer> organisationUnitIds, final Collection<DataElementOperand> operands )
{
- final Collection<DataElementOperand> sumOperands = filterOperands( operands, DataElement.AGGREGATION_OPERATOR_SUM );
- final Collection<DataElementOperand> averageOperands = filterOperands( operands, DataElement.AGGREGATION_OPERATOR_AVERAGE );
-
- final Map<DataElementOperand, Integer> sumOperandIndexMap = crossTabService.getOperandIndexMap( sumOperands );
- final Map<DataElementOperand, Integer> averageOperandIndexMap = crossTabService.getOperandIndexMap( averageOperands );
+ final Map<DataElementOperand, Integer> operandIndexMap = crossTabService.getOperandIndexMap( operands );
final Collection<Indicator> indicators = indicatorService.getIndicators( indicatorIds );
final Collection<Period> periods = periodService.getPeriods( periodIds );
@@ -166,18 +155,10 @@
OrganisationUnitHierarchy hierarchy = organisationUnitService.getOrganisationUnitHierarchy().prepareChildren( organisationUnitIds );
int count = 0;
- int level = 0;
-
- Map<DataElementOperand, Double> sumIntValueMap = null;
- Map<DataElementOperand, Double> averageIntValueMap = null;
-
- Map<String, Map<DataElementOperand, Double>> valueMapMap = null;
Map<DataElementOperand, Double> numeratorValueMap = null;
Map<DataElementOperand, Double> denominatorValueMap = null;
- PeriodType periodType = null;
-
double numeratorValue = 0.0;
double denominatorValue = 0.0;
@@ -190,22 +171,25 @@
final AggregatedIndicatorValue indicatorValue = new AggregatedIndicatorValue();
- for ( final OrganisationUnit unit : organisationUnits )
+ for ( final Period period : periods )
{
- level = aggregationCache.getLevelOfOrganisationUnit( unit.getId() );
+ final PeriodType periodType = period.getPeriodType();
- for ( final Period period : periods )
+ final Map<DataElementOperand, Integer> sumOperandIndexMap = getSumOperands( operands, periodType, operandIndexMap );
+ final Map<DataElementOperand, Integer> averageOperandIndexMap = getAvgOperands( operands, periodType, operandIndexMap );
+
+ for ( final OrganisationUnit unit : organisationUnits )
{
- sumIntValueMap = sumIntAggregator.getAggregatedValues( sumOperandIndexMap, period, unit, level, hierarchy );
- averageIntValueMap = averageIntAggregator.getAggregatedValues( averageOperandIndexMap, period, unit, level, hierarchy);
-
- valueMapMap = new HashMap<String, Map<DataElementOperand, Double>>( 2 );
+ final int level = aggregationCache.getLevelOfOrganisationUnit( unit.getId() );
+
+ final Map<DataElementOperand, Double> sumIntValueMap = sumIntAggregator.getAggregatedValues( sumOperandIndexMap, period, unit, level, hierarchy );
+ final Map<DataElementOperand, Double> averageIntValueMap = averageIntAggregator.getAggregatedValues( averageOperandIndexMap, period, unit, level, hierarchy);
+
+ final Map<String, Map<DataElementOperand, Double>> valueMapMap = new HashMap<String, Map<DataElementOperand, Double>>( 2 );
valueMapMap.put( DataElement.AGGREGATION_OPERATOR_SUM, sumIntValueMap );
valueMapMap.put( DataElement.AGGREGATION_OPERATOR_AVERAGE, averageIntValueMap );
- periodType = period.getPeriodType();
-
for ( final Indicator indicator : indicators )
{
// ---------------------------------------------------------
@@ -268,23 +252,38 @@
// Supportive methods
// -------------------------------------------------------------------------
- private Collection<DataElementOperand> filterOperands( final Collection<DataElementOperand> operands, final String aggregationOperator )
- {
- final Collection<DataElementOperand> filteredOperands = new ArrayList<DataElementOperand>();
-
- for ( final DataElementOperand operand : operands )
- {
- final DataElement dataElement = dataElementService.getDataElement( operand.getDataElementId() );
-
- if ( aggregationOperator.equals( dataElement.getAggregationOperator() ) )
- {
- filteredOperands.add( operand );
- }
- }
-
- return filteredOperands;
- }
-
+ private Map<DataElementOperand, Integer> getSumOperands( Collection<DataElementOperand> operands, PeriodType periodType, Map<DataElementOperand, Integer> operandIndexMap )
+ {
+ Map<DataElementOperand, Integer> sumOperandIndexMap = new HashMap<DataElementOperand, Integer>();
+
+ for ( final DataElementOperand operand : operands )
+ {
+ if ( operand.getAggregationOperator().equals( AGGREGATION_OPERATOR_SUM ) ||
+ ( operand.getAggregationOperator().equals( AGGREGATION_OPERATOR_AVERAGE ) && operand.getFrequencyOrder() >= periodType.getFrequencyOrder() ) )
+ {
+ sumOperandIndexMap.put( operand, operandIndexMap.get( operand ) );
+ }
+ }
+
+ return sumOperandIndexMap;
+ }
+
+ private Map<DataElementOperand, Integer> getAvgOperands( Collection<DataElementOperand> operands, PeriodType periodType, Map<DataElementOperand, Integer> operandIndexMap )
+ {
+ Map<DataElementOperand, Integer> avgOperandIndexMap = new HashMap<DataElementOperand, Integer>();
+
+ for ( final DataElementOperand operand : operands )
+ {
+ if ( operand.getAggregationOperator().equals( AGGREGATION_OPERATOR_AVERAGE ) &&
+ operand.getFrequencyOrder() < periodType.getFrequencyOrder() )
+ {
+ avgOperandIndexMap.put( operand, operandIndexMap.get( operand ) );
+ }
+ }
+
+ return avgOperandIndexMap;
+ }
+
private double getAnnualizationFactor( final Indicator indicator, final Period period )
{
double factor = 1.0;
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml 2010-06-08 19:47:40 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml 2010-06-14 16:27:39 +0000
@@ -165,8 +165,6 @@
ref="org.hisp.dhis.datamart.aggregation.dataelement.AverageIntAggregator"/>
<property name="crossTabService"
ref="org.hisp.dhis.datamart.crosstab.CrossTabService"/>
- <property name="dataElementService"
- ref="org.hisp.dhis.dataelement.DataElementService"/>
<property name="aggregationCache"
ref="org.hisp.dhis.datamart.aggregation.cache.AggregationCache"/>
<property name="systemSettingManager"
