dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #06513
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 2079: Applied direct summarizing of the average trick to aggregation service
------------------------------------------------------------
revno: 2079
committer: Lars <larshelg@larshelg-laptop>
branch nick: trunk
timestamp: Sun 2010-07-04 13:50:16 +0200
message:
Applied direct summarizing of the average trick to aggregation service
added:
dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/dataelement/AverageIntSingleValueDataElementAggregation.java
modified:
dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/DefaultAggregationService.java
dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/cache/AggregationCache.java
dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/cache/MemoryAggregationCache.java
dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/dataelement/AverageBoolDataElementAggregation.java
dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/dataelement/AverageIntDataElementAggregation.java
dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/dataelement/SumBoolDataElementAggregation.java
dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/dataelement/SumIntDataElementAggregation.java
dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/resources/META-INF/dhis/beans.xml
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntSingleValueAggregator.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/DefaultAggregationService.java'
--- dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/DefaultAggregationService.java 2010-07-04 10:48:51 +0000
+++ dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/DefaultAggregationService.java 2010-07-04 11:50:16 +0000
@@ -37,6 +37,7 @@
import org.hisp.dhis.dataelement.DataElementCategoryOptionCombo;
import org.hisp.dhis.indicator.Indicator;
import org.hisp.dhis.organisationunit.OrganisationUnit;
+import static org.hisp.dhis.system.util.DateUtils.*;
import static org.hisp.dhis.dataelement.DataElement.*;
@@ -72,6 +73,13 @@
this.averageIntDataElementAggregation = averageIntDataElementAggregation;
}
+ private AbstractDataElementAggregation averageIntSingleValueAggregation;
+
+ public void setAverageIntSingleValueAggregation( AbstractDataElementAggregation averageIntSingleValueAggregation )
+ {
+ this.averageIntSingleValueAggregation = averageIntSingleValueAggregation;
+ }
+
private AbstractDataElementAggregation averageBoolDataElementAggregation;
public void setAverageBoolDataElementAggregation( AbstractDataElementAggregation averageBoolDataElementAggregation )
@@ -101,7 +109,7 @@
OrganisationUnit organisationUnit )
{
AbstractDataElementAggregation dataElementAggregation =
- getInstance( dataElement.getType(), dataElement.getAggregationOperator() );
+ getInstance( dataElement.getType(), dataElement.getAggregationOperator(), startDate, endDate, dataElement );
return dataElementAggregation.getAggregatedValue( dataElement, optionCombo, startDate, endDate, organisationUnit );
}
@@ -137,7 +145,7 @@
// Supportive methods
// -------------------------------------------------------------------------
- private AbstractDataElementAggregation getInstance( String valueType, String aggregationOperator )
+ private AbstractDataElementAggregation getInstance( String valueType, String aggregationOperator, Date startDate, Date endDate, DataElement dataElement )
{
if ( valueType.equals( VALUE_TYPE_INT ) && aggregationOperator.equals( AGGREGATION_OPERATOR_SUM ) )
{
@@ -147,6 +155,10 @@
{
return sumBoolDataElementAggregation;
}
+ else if ( valueType.equals( VALUE_TYPE_INT ) && aggregationOperator.equals( AGGREGATION_OPERATOR_AVERAGE ) && dataElement.getFrequencyOrder() >= getDaysInclusive( startDate, endDate ) )
+ {
+ return averageIntSingleValueAggregation;
+ }
else if ( valueType.equals( VALUE_TYPE_INT ) && aggregationOperator.equals( AGGREGATION_OPERATOR_AVERAGE ) )
{
return averageIntDataElementAggregation;
=== modified file 'dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/cache/AggregationCache.java'
--- dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/cache/AggregationCache.java 2010-06-27 15:34:05 +0000
+++ dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/cache/AggregationCache.java 2010-07-04 11:50:16 +0000
@@ -46,7 +46,7 @@
Period getPeriod( int periodId );
- Collection<Integer> getPeriodIds( Date startDate, Date endDate );
+ Collection<Integer> getIntersectingPeriodIds( Date startDate, Date endDate );
double getAggregatedDataValue( DataElement dataElement, DataElementCategoryOptionCombo optionCombo, Date startDate, Date endDate, OrganisationUnit organisationUnit );
=== modified file 'dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/cache/MemoryAggregationCache.java'
--- dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/cache/MemoryAggregationCache.java 2010-06-27 15:34:05 +0000
+++ dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/cache/MemoryAggregationCache.java 2010-07-04 11:50:16 +0000
@@ -114,7 +114,7 @@
return period;
}
- public Collection<Integer> getPeriodIds( Date startDate, Date endDate )
+ public Collection<Integer> getIntersectingPeriodIds( Date startDate, Date endDate )
{
String key = startDate.toString() + SEPARATOR + endDate.toString();
=== modified file 'dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/dataelement/AverageBoolDataElementAggregation.java'
--- dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/dataelement/AverageBoolDataElementAggregation.java 2010-06-08 19:47:40 +0000
+++ dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/dataelement/AverageBoolDataElementAggregation.java 2010-07-04 11:50:16 +0000
@@ -70,7 +70,7 @@
{
OrganisationUnitHierarchy hierarchy = aggregationCache.getOrganisationUnitHierarchy();
- Collection<Integer> periods = aggregationCache.getPeriodIds( startDate, endDate );
+ Collection<Integer> periods = aggregationCache.getIntersectingPeriodIds( startDate, endDate );
Collection<DataValue> values = aggregationStore.getDataValues( hierarchy.getChildren( organisationUnitId ), dataElementId, optionComboId, periods );
=== modified file 'dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/dataelement/AverageIntDataElementAggregation.java'
--- dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/dataelement/AverageIntDataElementAggregation.java 2010-06-27 15:34:05 +0000
+++ dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/dataelement/AverageIntDataElementAggregation.java 2010-07-04 11:50:16 +0000
@@ -96,7 +96,7 @@
protected Collection<DataValue> getDataValues( int dataElementId, int optionComboId, int organisationUnitId,
Date startDate, Date endDate )
{
- Collection<Integer> periods = aggregationCache.getPeriodIds( startDate, endDate );
+ Collection<Integer> periods = aggregationCache.getIntersectingPeriodIds( startDate, endDate );
Collection<DataValue> values = aggregationStore.getDataValues( organisationUnitId, dataElementId, optionComboId, periods );
@@ -128,7 +128,7 @@
* aggregation period
* @param aggregationEndDate The original end date of the entire aggregation
* period
- * @return The numerator and denominator of the AVERAGE value
+ * @return The AVERAGE value.
*/
protected double[] getAggregateOfValues( Collection<DataValue> dataValues, Date startDate, Date endDate,
Date aggregationStartDate, Date aggregationEndDate )
=== added file 'dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/dataelement/AverageIntSingleValueDataElementAggregation.java'
--- dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/dataelement/AverageIntSingleValueDataElementAggregation.java 1970-01-01 00:00:00 +0000
+++ dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/dataelement/AverageIntSingleValueDataElementAggregation.java 2010-07-04 11:50:16 +0000
@@ -0,0 +1,140 @@
+package org.hisp.dhis.aggregation.impl.dataelement;
+
+/*
+ * Copyright (c) 2004-2010, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+import static org.hisp.dhis.system.util.DateUtils.getDays;
+
+import java.util.Collection;
+import java.util.Date;
+
+import org.hisp.dhis.aggregation.AggregationService;
+import org.hisp.dhis.dataelement.DataElement;
+import org.hisp.dhis.dataelement.DataElementCategoryOptionCombo;
+import org.hisp.dhis.datavalue.DataValue;
+import org.hisp.dhis.organisationunit.OrganisationUnit;
+import org.hisp.dhis.organisationunit.OrganisationUnitHierarchy;
+import org.hisp.dhis.period.Period;
+
+/**
+ * @author Lars Helge Overland
+ * @version $Id: SumBoolDataElementAggregation.java 4753 2008-03-14 12:48:50Z larshelg $
+ */
+public class AverageIntSingleValueDataElementAggregation
+ extends AbstractDataElementAggregation
+{
+ public double getAggregatedValue( DataElement dataElement, DataElementCategoryOptionCombo optionCombo, Date aggregationStartDate, Date aggregationEndDate,
+ OrganisationUnit organisationUnit )
+ {
+ double[] sums = getSumAndRelevantDays( dataElement.getId(), optionCombo.getId(), aggregationStartDate, aggregationEndDate,
+ organisationUnit.getId() );
+
+ if ( sums[1] > 0 )
+ {
+ return sums[0];
+ }
+ else
+ {
+ return AggregationService.NO_VALUES_REGISTERED;
+ }
+ }
+
+ protected Collection<DataValue> getDataValues( int dataElementId, int optionComboId, int organisationUnitId,
+ Date startDate, Date endDate )
+ {
+ OrganisationUnitHierarchy hierarchy = aggregationCache.getOrganisationUnitHierarchy();
+
+ Collection<Integer> periods = aggregationCache.getIntersectingPeriodIds( startDate, endDate );
+
+ Collection<DataValue> values = aggregationStore.getDataValues( hierarchy.getChildren( organisationUnitId ), dataElementId, optionComboId, periods );
+
+ return values;
+ }
+
+ /**
+ * The main performance disadvantage of average aggregation operations is that
+ * the average must be calculated for each organisation unit individually, and
+ * then summarized, in contrast to sum operations where the sum can be calculated
+ * directly for all organisation units. Still, in cases where the aggregation
+ * period duration is shorter than the data element period type duration, there
+ * can only be one value registered for each organisation unit for that data
+ * element. This implies that there is no need to calculate the average and
+ * that the aggregate can be calculated directly, improving performance
+ * dramatically. This method is performs the described behaviour by taking
+ * the sum directly using whatever value found.
+ *
+ * @param dataValues The datavalues to aggregate
+ * @param startDate Start date of the period to aggregate over
+ * @param endDate End date of the period to aggregate over
+ * @param aggregationStartDate The original start date of the entire
+ * aggregation period
+ * @param aggregationEndDate The original end date of the entire aggregation
+ * period
+ */
+ protected double[] getAggregateOfValues( Collection<DataValue> dataValues, Date startDate, Date endDate,
+ Date aggregationStartDate, Date aggregationEndDate )
+ {
+ double totalSum = 0;
+ double totalRelevantDays = 0;
+
+ for ( DataValue dataValue : dataValues )
+ {
+ Period currentPeriod = aggregationCache.getPeriod( dataValue.getPeriod().getId() );
+ Date currentStartDate = currentPeriod.getStartDate();
+ Date currentEndDate = currentPeriod.getEndDate();
+
+ double value = 0;
+
+ try
+ {
+ value = Double.parseDouble( dataValue.getValue() );
+ }
+ catch ( Exception ex )
+ {
+ }
+
+ double currentPeriodDuration = ( getDays( currentEndDate ) - getDays( currentStartDate ) );
+
+ if ( currentPeriodDuration > 0 )
+ {
+ long relevantDays = 0;
+
+ if ( currentStartDate.compareTo( endDate ) <= 0 && currentEndDate.compareTo( startDate ) >= 0 ) // Value is intersecting
+ {
+ relevantDays = getDays( endDate ) - getDays( startDate );
+ totalSum += value;
+ }
+
+ totalRelevantDays += relevantDays;
+ }
+ }
+
+ double[] fraction = { totalSum, totalRelevantDays };
+
+ return fraction;
+ }
+}
=== modified file 'dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/dataelement/SumBoolDataElementAggregation.java'
--- dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/dataelement/SumBoolDataElementAggregation.java 2010-06-27 15:34:05 +0000
+++ dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/dataelement/SumBoolDataElementAggregation.java 2010-07-04 11:50:16 +0000
@@ -41,7 +41,6 @@
import static org.hisp.dhis.system.util.MathUtils.getFloor;
import static org.hisp.dhis.system.util.DateUtils.getDays;
-
/**
* @author Lars Helge Overland
* @version $Id: SumBoolDataElementAggregation.java 4753 2008-03-14 12:48:50Z larshelg $
@@ -70,7 +69,7 @@
{
OrganisationUnitHierarchy hierarchy = aggregationCache.getOrganisationUnitHierarchy();
- Collection<Integer> periods = aggregationCache.getPeriodIds( startDate, endDate );
+ Collection<Integer> periods = aggregationCache.getIntersectingPeriodIds( startDate, endDate );
Collection<DataValue> values = aggregationStore.getDataValues( hierarchy.getChildren( organisationUnitId ), dataElementId, optionComboId, periods );
=== modified file 'dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/dataelement/SumIntDataElementAggregation.java'
--- dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/dataelement/SumIntDataElementAggregation.java 2010-06-08 19:47:40 +0000
+++ dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/java/org/hisp/dhis/aggregation/impl/dataelement/SumIntDataElementAggregation.java 2010-07-04 11:50:16 +0000
@@ -68,7 +68,7 @@
{
OrganisationUnitHierarchy hierarchy = aggregationCache.getOrganisationUnitHierarchy();
- Collection<Integer> periods = aggregationCache.getPeriodIds( startDate, endDate );
+ Collection<Integer> periods = aggregationCache.getIntersectingPeriodIds( startDate, endDate );
Collection<DataValue> values = aggregationStore.getDataValues( hierarchy.getChildren( organisationUnitId ), dataElementId, optionComboId, periods );
=== modified file 'dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/resources/META-INF/dhis/beans.xml'
--- dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/resources/META-INF/dhis/beans.xml 2010-06-27 15:34:05 +0000
+++ dhis-2/dhis-services/dhis-service-aggregationengine-default/src/main/resources/META-INF/dhis/beans.xml 2010-07-04 11:50:16 +0000
@@ -21,6 +21,8 @@
ref="org.hisp.dhis.aggregation.impl.dataelement.SumBoolDataElementAggregation"/>
<property name="averageIntDataElementAggregation"
ref="org.hisp.dhis.aggregation.impl.dataelement.AverageIntDataElementAggregation"/>
+ <property name="averageIntSingleValueAggregation"
+ ref="org.hisp.dhis.aggregation.impl.dataelement.AverageIntSingleValueDataElementAggregation"/>
<property name="averageBoolDataElementAggregation"
ref="org.hisp.dhis.aggregation.impl.dataelement.AverageBoolDataElementAggregation"/>
<property name="indicatorAggregation"
@@ -63,6 +65,14 @@
ref="org.hisp.dhis.aggregation.impl.cache.AggregationCache"/>
</bean>
+ <bean id="org.hisp.dhis.aggregation.impl.dataelement.AverageIntSingleValueDataElementAggregation"
+ class="org.hisp.dhis.aggregation.impl.dataelement.AverageIntSingleValueDataElementAggregation">
+ <property name="aggregationStore"
+ ref="org.hisp.dhis.aggregation.AggregationStore"/>
+ <property name="aggregationCache"
+ ref="org.hisp.dhis.aggregation.impl.cache.AggregationCache"/>
+ </bean>
+
<bean id="org.hisp.dhis.aggregation.impl.dataelement.AverageBoolDataElementAggregation"
class="org.hisp.dhis.aggregation.impl.dataelement.AverageBoolDataElementAggregation">
<property name="aggregationStore"
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntSingleValueAggregator.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntSingleValueAggregator.java 2010-07-04 11:11:30 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntSingleValueAggregator.java 2010-07-04 11:50:16 +0000
@@ -93,7 +93,7 @@
final Map<DataElementOperand, double[]> entries = getAggregate( crossTabValues, period.getStartDate(),
period.getEndDate(), period.getStartDate(), period.getEndDate(), unitLevel ); // <Operand, [total value, total relevant days]>
- final Map<DataElementOperand, Double> values = new HashMap<DataElementOperand, Double>(); // <Operand, total value>
+ final Map<DataElementOperand, Double> values = new HashMap<DataElementOperand, Double>( entries.size() ); // <Operand, total value>
for ( final Entry<DataElementOperand, double[]> entry : entries.entrySet() )
{
