dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #18807
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 7982: Data mart: made aggregation method faster by retrieving larger chunks of data from crosstab table...
------------------------------------------------------------
revno: 7982
committer: Lars Helge Øverland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Wed 2012-08-29 16:39:44 +0200
message:
Data mart: made aggregation method faster by retrieving larger chunks of data from crosstab table. Improves data mart performance by approx 30 percent.
removed:
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntSingleValueAggregator.java
added:
dhis-2/dhis-api/src/main/java/org/hisp/dhis/period/PeriodHierarchy.java
modified:
dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataelement/DataElementOperand.java
dhis-2/dhis-api/src/main/java/org/hisp/dhis/period/PeriodService.java
dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/period/DefaultPeriodService.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/OrgUnitOperand.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/cache/AggregationCache.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/cache/MemoryAggregationCache.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageBoolAggregator.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntAggregator.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/DataElementAggregator.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumBoolAggregator.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumIntAggregator.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/CrossTabStore.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/JDBCCrossTabStore.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DataElementDataMart.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DefaultDataElementDataMart.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml
dhis-2/dhis-services/dhis-service-datamart-default/src/test/java/org/hisp/dhis/datamart/DataMartServiceTest.java
dhis-2/dhis-services/dhis-service-datamart-default/src/test/java/org/hisp/dhis/datamart/crosstab/CrossTabServiceTest.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataelement/DataElementOperand.java'
--- dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataelement/DataElementOperand.java 2012-08-25 11:47:17 +0000
+++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataelement/DataElementOperand.java 2012-08-29 14:39:44 +0000
@@ -486,8 +486,8 @@
final int prime = 31;
int result = 1;
- result = prime * result + ((categoryOptionCombo == null) ? 0 : categoryOptionCombo.hashCode());
- result = prime * result + ((dataElement == null) ? 0 : dataElement.hashCode());
+ result = prime * result + ( ( categoryOptionCombo == null) ? 0 : categoryOptionCombo.hashCode() );
+ result = prime * result + ( ( dataElement == null) ? 0 : dataElement.hashCode() );
result = prime * result + dataElementId;
result = prime * result + optionComboId;
=== added file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/period/PeriodHierarchy.java'
--- dhis-2/dhis-api/src/main/java/org/hisp/dhis/period/PeriodHierarchy.java 1970-01-01 00:00:00 +0000
+++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/period/PeriodHierarchy.java 2012-08-29 14:39:44 +0000
@@ -0,0 +1,68 @@
+package org.hisp.dhis.period;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class PeriodHierarchy
+{
+ private Map<Integer, Set<Integer>> intersectingPeriods = new HashMap<Integer, Set<Integer>>();
+
+ private Map<Integer, Set<Integer>> periodsBetween = new HashMap<Integer, Set<Integer>>();
+
+ public Map<Integer, Set<Integer>> getIntersectingPeriods()
+ {
+ return intersectingPeriods;
+ }
+
+ public Set<Integer> getIntersectingPeriods( Period period )
+ {
+ return intersectingPeriods.get( period.getId() );
+ }
+
+ public Set<Integer> getIntersectingPeriods( Collection<Period> periods )
+ {
+ periods = new HashSet<Period>( periods );
+
+ Set<Integer> set = new HashSet<Integer>();
+
+ for ( Period period : periods )
+ {
+ if ( intersectingPeriods.containsKey( period.getId() ) )
+ {
+ set.addAll( intersectingPeriods.get( period.getId() ) );
+ }
+ }
+
+ return set;
+ }
+
+ public Map<Integer, Set<Integer>> getPeriodsBetween()
+ {
+ return periodsBetween;
+ }
+
+ public Set<Integer> getPeriodsBetween( Period period )
+ {
+ return periodsBetween.get( period.getId() );
+ }
+
+ public Set<Integer> getPeriodsBetween( Collection<Period> periods )
+ {
+ periods = new HashSet<Period>( periods );
+
+ Set<Integer> set = new HashSet<Integer>();
+
+ for ( Period period : periods )
+ {
+ if ( periodsBetween.containsKey( period.getId() ) )
+ {
+ set.addAll( periodsBetween.get( period.getId() ) );
+ }
+ }
+
+ return set;
+ }
+}
=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/period/PeriodService.java'
--- dhis-2/dhis-api/src/main/java/org/hisp/dhis/period/PeriodService.java 2012-04-21 08:25:31 +0000
+++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/period/PeriodService.java 2012-08-29 14:39:44 +0000
@@ -30,6 +30,7 @@
import java.util.Collection;
import java.util.Date;
import java.util.List;
+import java.util.Set;
import org.hisp.dhis.dataelement.DataElement;
import org.hisp.dhis.i18n.I18nFormat;
@@ -271,6 +272,14 @@
*/
Period reloadPeriod( Period period );
+ /**
+ * Returns a PeriodHierarchy instance.
+ *
+ * @param periods the Periods to include in the PeriodHierarchy.
+ * @return a PeriodHierarchy instance.
+ */
+ PeriodHierarchy getPeriodHierarchy( Collection<Period> periods );
+
// -------------------------------------------------------------------------
// PeriodType
// -------------------------------------------------------------------------
=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/period/DefaultPeriodService.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/period/DefaultPeriodService.java 2012-04-21 08:25:31 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/period/DefaultPeriodService.java 2012-08-29 14:39:44 +0000
@@ -27,6 +27,17 @@
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
+import static org.hisp.dhis.system.util.ConversionUtils.getIdentifiers;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
import org.hisp.dhis.dataelement.DataElement;
import org.hisp.dhis.i18n.I18nFormat;
import org.hisp.dhis.organisationunit.OrganisationUnit;
@@ -35,8 +46,6 @@
import org.hisp.dhis.system.util.FilterUtils;
import org.springframework.transaction.annotation.Transactional;
-import java.util.*;
-
/**
* @author Kristian Nordal
* @version $Id: DefaultPeriodService.java 5983 2008-10-17 17:42:44Z larshelg $
@@ -258,6 +267,19 @@
{
return periodStore.reloadForceAddPeriod( period );
}
+
+ public PeriodHierarchy getPeriodHierarchy( Collection<Period> periods )
+ {
+ PeriodHierarchy hierarchy = new PeriodHierarchy();
+
+ for ( Period period : periods )
+ {
+ hierarchy.getIntersectingPeriods().put( period.getId(), new HashSet<Integer>( getIdentifiers( Period.class, getIntersectingPeriods( period.getStartDate(), period.getEndDate() ) ) ) );
+ hierarchy.getPeriodsBetween().put( period.getId(), new HashSet<Integer>( getIdentifiers( Period.class, getPeriodsBetweenDates( period.getStartDate(), period.getEndDate() ) ) ) );
+ }
+
+ return hierarchy;
+ }
// -------------------------------------------------------------------------
// PeriodType
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/OrgUnitOperand.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/OrgUnitOperand.java 2012-08-25 11:49:05 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/OrgUnitOperand.java 2012-08-29 14:39:44 +0000
@@ -1,7 +1,41 @@
package org.hisp.dhis.datamart;
+/*
+ * Copyright (c) 2004-2012, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * @author Lars Helge Overland
+ */
public class OrgUnitOperand
{
+ private int periodId;
+
+ private int periodTypeId;
+
private int orgUnitId;
private int orgUnitGroupId;
@@ -12,13 +46,35 @@
{
}
- public OrgUnitOperand( int orgUnitId, int orgUnitGroupId, double value )
+ public OrgUnitOperand( int periodId, int periodTypeId, int orgUnitId, int orgUnitGroupId, double value )
{
+ this.periodId = periodId;
+ this.periodTypeId = periodTypeId;
this.orgUnitId = orgUnitId;
this.orgUnitGroupId = orgUnitGroupId;
this.value = value;
}
+ public int getPeriodId()
+ {
+ return periodId;
+ }
+
+ public void setPeriodId( int periodId )
+ {
+ this.periodId = periodId;
+ }
+
+ public int getPeriodTypeId()
+ {
+ return periodTypeId;
+ }
+
+ public void setPeriodTypeId( int periodTypeId )
+ {
+ this.periodTypeId = periodTypeId;
+ }
+
public int getOrgUnitId()
{
return orgUnitId;
@@ -65,5 +121,11 @@
final OrgUnitOperand other = (OrgUnitOperand) obj;
return orgUnitId == other.orgUnitId && orgUnitGroupId == other.orgUnitGroupId;
- }
+ }
+
+ @Override
+ public String toString()
+ {
+ return "[period: " + periodId + ", org unit: " + orgUnitId + ", group: " + orgUnitGroupId + ", value: " + value + "]";
+ }
}
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/cache/AggregationCache.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/cache/AggregationCache.java 2012-03-15 16:04:20 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/cache/AggregationCache.java 2012-08-29 14:39:44 +0000
@@ -29,7 +29,9 @@
import java.util.Collection;
import java.util.Date;
+import java.util.Set;
+import org.hisp.dhis.dataelement.DataElementOperand;
import org.hisp.dhis.period.Period;
import org.hisp.dhis.period.PeriodType;
@@ -49,5 +51,7 @@
int getLevelOfOrganisationUnit( int id );
+ void filterForAggregationLevel( Set<Integer> organisationUnits, DataElementOperand operand, int unitLevel );
+
void clearCache();
}
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/cache/MemoryAggregationCache.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/cache/MemoryAggregationCache.java 2012-08-25 11:47:17 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/cache/MemoryAggregationCache.java 2012-08-29 14:39:44 +0000
@@ -30,8 +30,11 @@
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.Map;
+import java.util.Set;
+import org.hisp.dhis.dataelement.DataElementOperand;
import org.hisp.dhis.organisationunit.OrganisationUnitService;
import org.hisp.dhis.period.Period;
import org.hisp.dhis.period.PeriodService;
@@ -172,6 +175,23 @@
return level;
}
+ public void filterForAggregationLevel( Set<Integer> organisationUnits, DataElementOperand operand, int unitLevel )
+ {
+ final Iterator<Integer> iter = organisationUnits.iterator();
+
+ while ( iter.hasNext() )
+ {
+ final Integer orgUnitId = iter.next();
+
+ final int dataValueLevel = operand.isHasAggregationLevels() ? getLevelOfOrganisationUnit( orgUnitId ) : 0;
+
+ if ( operand.isHasAggregationLevels() && !operand.aggregationLevelIsValid( unitLevel, dataValueLevel ) )
+ {
+ iter.remove();
+ }
+ }
+ }
+
public void clearCache()
{
intersectingPeriodCache.clear();
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageBoolAggregator.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageBoolAggregator.java 2012-08-25 11:47:17 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageBoolAggregator.java 2012-08-29 14:39:44 +0000
@@ -35,15 +35,18 @@
import java.util.Collection;
import java.util.Date;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.hisp.dhis.dataelement.DataElementOperand;
-import org.hisp.dhis.datamart.CrossTabDataValue;
import org.hisp.dhis.datamart.OrgUnitOperand;
import org.hisp.dhis.datamart.aggregation.cache.AggregationCache;
import org.hisp.dhis.datamart.crosstab.CrossTabService;
+import org.hisp.dhis.datamart.crosstab.jdbc.CrossTabStore;
import org.hisp.dhis.organisationunit.OrganisationUnitGroup;
import org.hisp.dhis.organisationunit.OrganisationUnitHierarchy;
import org.hisp.dhis.period.Period;
+import org.hisp.dhis.period.PeriodHierarchy;
import org.hisp.dhis.period.PeriodType;
import org.hisp.dhis.system.util.MathUtils;
@@ -75,11 +78,12 @@
// DataElementAggregator implementation
// -------------------------------------------------------------------------
- public List<OrgUnitOperand> getAggregatedValues( DataElementOperand operand,
- Period period, Collection<Integer> organisationUnits, Collection<OrganisationUnitGroup> organisationUnitGroups, OrganisationUnitHierarchy hierarchy, String key )
+ public List<OrgUnitOperand> getAggregatedValues( DataElementOperand operand, Collection<Period> periods,
+ Collection<Integer> organisationUnits, Collection<OrganisationUnitGroup> organisationUnitGroups,
+ PeriodHierarchy periodHierarchy, OrganisationUnitHierarchy orgUnitHierarchy, String key )
{
- final Collection<CrossTabDataValue> crossTabValues = crossTabService.getCrossTabDataValues( operand,
- aggregationCache.getIntersectingPeriods( period.getStartDate(), period.getEndDate() ), hierarchy.getChildren( organisationUnits ), key );
+ final Map<String, String> crossTabValues = crossTabService.getCrossTabDataValues( operand,
+ periodHierarchy.getIntersectingPeriods( periods ), orgUnitHierarchy.getChildren( organisationUnits ), key );
final List<OrgUnitOperand> values = new ArrayList<OrgUnitOperand>();
@@ -87,38 +91,48 @@
{
return values;
}
-
- for ( final Integer organisationUnit : organisationUnits )
+
+ for ( Period period : periods )
{
- final int unitLevel = operand.isHasAggregationLevels() ? aggregationCache.getLevelOfOrganisationUnit( organisationUnit ) : 0;
-
- for ( OrganisationUnitGroup group : organisationUnitGroups )
+ final PeriodType periodType = period.getPeriodType();
+
+ final Collection<Integer> intersectingPeriods = periodHierarchy.getIntersectingPeriods( period );
+
+ for ( final Integer organisationUnit : organisationUnits )
{
- final Collection<Integer> orgUnitChildren = hierarchy.getChildren( organisationUnit, group );
-
- double totalValue = 0.0;
- double totalRelevantDays = 0.0;
-
- for ( Integer orgUnitChild : orgUnitChildren )
- {
- final int dataValueLevel = operand.isHasAggregationLevels() ? aggregationCache.getLevelOfOrganisationUnit( orgUnitChild ) : 0;
-
- if ( operand.isHasAggregationLevels() && !operand.aggregationLevelIsValid( unitLevel, dataValueLevel ) )
- {
- continue;
- }
-
- final double[] entry = getAggregate( orgUnitChild, crossTabValues, period.getStartDate(), period.getEndDate(), unitLevel ); // <Operand, [total value, total relevant days]>
-
- totalValue += entry[0];
- totalRelevantDays += entry[1];
- }
-
- if ( !MathUtils.isZero( totalRelevantDays ) )
- {
- double average = totalValue / totalRelevantDays * 100; // Percentage value
-
- values.add( new OrgUnitOperand( organisationUnit, group != null ? group.getId() : 0, average ) );
+ final int unitLevel = operand.isHasAggregationLevels() ? aggregationCache.getLevelOfOrganisationUnit( organisationUnit ) : 0;
+
+ for ( OrganisationUnitGroup group : organisationUnitGroups )
+ {
+ final Set<Integer> orgUnitChildren = orgUnitHierarchy.getChildren( organisationUnit, group );
+
+ aggregationCache.filterForAggregationLevel( orgUnitChildren, operand, unitLevel );
+
+ double totalValue = 0d;
+ double totalRelevantDays = 0d;
+
+ for ( Integer orgUnitChild : orgUnitChildren )
+ {
+ for ( Integer intersectingPeriod : intersectingPeriods )
+ {
+ final String val = crossTabValues.get( intersectingPeriod + CrossTabStore.SEPARATOR + orgUnitChild );
+
+ if ( val != null )
+ {
+ final double[] entry = getAggregate( orgUnitChild, period, val, period.getStartDate(), period.getEndDate(), unitLevel ); // <Operand, [total value, total relevant days]>
+
+ totalValue += entry[0];
+ totalRelevantDays += entry[1];
+ }
+ }
+ }
+
+ if ( !MathUtils.isZero( totalRelevantDays ) )
+ {
+ double average = totalValue / totalRelevantDays * 100; // Percentage value
+
+ values.add( new OrgUnitOperand( period.getId(), periodType.getId(), organisationUnit, group != null ? group.getId() : 0, average ) );
+ }
}
}
}
@@ -126,69 +140,50 @@
return values;
}
- private double[] getAggregate( int organisationUnit, Collection<CrossTabDataValue> crossTabValues,
- Date startDate, Date endDate, int unitLevel )
+ private double[] getAggregate( int organisationUnit, Period period, String val, Date startDate, Date endDate, int unitLevel )
{
- double totalValue = 0.0;
- double totalRelevantDays = 0.0;
-
- for ( final CrossTabDataValue crossTabValue : crossTabValues )
+ double value = 0.0;
+ double relevantDays = 0.0;
+
+ final Date currentStartDate = period.getStartDate();
+ final Date currentEndDate = period.getEndDate();
+
+ final double duration = getDaysInclusive( currentStartDate, currentEndDate );
+
+ if ( duration > 0 )
{
- final Period period = aggregationCache.getPeriod( crossTabValue.getPeriodId() );
-
- final Date currentStartDate = period.getStartDate();
- final Date currentEndDate = period.getEndDate();
-
- final double duration = getDaysInclusive( currentStartDate, currentEndDate );
-
- if ( duration > 0 )
- {
- double value = 0.0;
- double relevantDays = 0.0;
-
- String val = crossTabValue.getValueMap().get( organisationUnit );
-
- if ( val == null )
- {
- continue;
- }
-
- if ( currentStartDate.compareTo( startDate ) >= 0 && currentEndDate.compareTo( endDate ) <= 0 ) // Value is within period
- {
- relevantDays = getDaysInclusive( currentStartDate, currentEndDate );
- }
- else if ( currentStartDate.compareTo( startDate ) <= 0 && currentEndDate.compareTo( endDate ) >= 0 ) // Value spans whole period
- {
- relevantDays = getDaysInclusive( startDate, endDate );
- }
- else if ( currentStartDate.compareTo( startDate ) <= 0 && currentEndDate.compareTo( startDate ) >= 0 // Value spans period start
- && currentEndDate.compareTo( endDate ) <= 0 )
- {
- relevantDays = getDaysInclusive( startDate, currentEndDate );
- }
- else if ( currentStartDate.compareTo( startDate ) >= 0 && currentStartDate.compareTo( endDate ) <= 0 // Value spans period end
- && currentEndDate.compareTo( endDate ) >= 0 )
- {
- relevantDays = getDaysInclusive( currentStartDate, endDate );
- }
-
- if ( val.toLowerCase().equals( TRUE ) )
- {
- value = relevantDays;
- }
-
- totalValue += value;
- totalRelevantDays += relevantDays;
+ if ( currentStartDate.compareTo( startDate ) >= 0 && currentEndDate.compareTo( endDate ) <= 0 ) // Value is within period
+ {
+ relevantDays = getDaysInclusive( currentStartDate, currentEndDate );
+ }
+ else if ( currentStartDate.compareTo( startDate ) <= 0 && currentEndDate.compareTo( endDate ) >= 0 ) // Value spans whole period
+ {
+ relevantDays = getDaysInclusive( startDate, endDate );
+ }
+ else if ( currentStartDate.compareTo( startDate ) <= 0 && currentEndDate.compareTo( startDate ) >= 0 // Value spans period start
+ && currentEndDate.compareTo( endDate ) <= 0 )
+ {
+ relevantDays = getDaysInclusive( startDate, currentEndDate );
+ }
+ else if ( currentStartDate.compareTo( startDate ) >= 0 && currentStartDate.compareTo( endDate ) <= 0 // Value spans period end
+ && currentEndDate.compareTo( endDate ) >= 0 )
+ {
+ relevantDays = getDaysInclusive( currentStartDate, endDate );
+ }
+
+ if ( val.toLowerCase().equals( TRUE ) )
+ {
+ value = relevantDays;
}
}
- final double[] values = { totalValue, totalRelevantDays };
+ final double[] values = { value, relevantDays };
return values;
}
- public boolean isApplicable( DataElementOperand operand, PeriodType periodType )
+ public boolean isApplicable( DataElementOperand operand )
{
- return ( operand.getValueType().equals( VALUE_TYPE_BOOL ) && operand.getAggregationOperator().equals( AGGREGATION_OPERATOR_AVERAGE ) );
+ return operand.getValueType().equals( VALUE_TYPE_BOOL ) && operand.getAggregationOperator().equals( AGGREGATION_OPERATOR_AVERAGE );
}
}
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntAggregator.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntAggregator.java 2012-08-25 11:47:17 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntAggregator.java 2012-08-29 14:39:44 +0000
@@ -35,17 +35,18 @@
import java.util.Collection;
import java.util.Date;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.hisp.dhis.dataelement.DataElementOperand;
-import org.hisp.dhis.datamart.CrossTabDataValue;
import org.hisp.dhis.datamart.OrgUnitOperand;
import org.hisp.dhis.datamart.aggregation.cache.AggregationCache;
import org.hisp.dhis.datamart.crosstab.CrossTabService;
+import org.hisp.dhis.datamart.crosstab.jdbc.CrossTabStore;
import org.hisp.dhis.organisationunit.OrganisationUnitGroup;
import org.hisp.dhis.organisationunit.OrganisationUnitHierarchy;
import org.hisp.dhis.period.Period;
+import org.hisp.dhis.period.PeriodHierarchy;
import org.hisp.dhis.period.PeriodType;
import org.hisp.dhis.system.util.MathUtils;
@@ -55,8 +56,6 @@
public class AverageIntAggregator
implements DataElementAggregator
{
- private static final Log log = LogFactory.getLog( AverageIntAggregator.class );
-
// -------------------------------------------------------------------------
// Dependencies
// -------------------------------------------------------------------------
@@ -79,11 +78,12 @@
// DataElementAggregator implementation
// -------------------------------------------------------------------------
- public List<OrgUnitOperand> getAggregatedValues( DataElementOperand operand, Period period,
- Collection<Integer> organisationUnits, Collection<OrganisationUnitGroup> organisationUnitGroups, OrganisationUnitHierarchy hierarchy, String key )
+ public List<OrgUnitOperand> getAggregatedValues( DataElementOperand operand, Collection<Period> periods,
+ Collection<Integer> organisationUnits, Collection<OrganisationUnitGroup> organisationUnitGroups,
+ PeriodHierarchy periodHierarchy, OrganisationUnitHierarchy orgUnitHierarchy, String key )
{
- final Collection<CrossTabDataValue> crossTabValues = crossTabService.getCrossTabDataValues( operand,
- aggregationCache.getIntersectingPeriods( period.getStartDate(), period.getEndDate() ), hierarchy.getChildren( organisationUnits ), key );
+ final Map<String, String> crossTabValues = crossTabService.getCrossTabDataValues( operand,
+ periodHierarchy.getIntersectingPeriods( periods ), orgUnitHierarchy.getChildren( organisationUnits ), key );
final List<OrgUnitOperand> values = new ArrayList<OrgUnitOperand>();
@@ -91,117 +91,105 @@
{
return values;
}
-
- for ( final Integer organisationUnit : organisationUnits )
+
+ for ( Period period : periods )
{
- final int unitLevel = operand.isHasAggregationLevels() ? aggregationCache.getLevelOfOrganisationUnit( organisationUnit ) : 0;
-
- for ( OrganisationUnitGroup group : organisationUnitGroups )
+ final PeriodType periodType = period.getPeriodType();
+
+ final Collection<Integer> intersectingPeriods = periodHierarchy.getIntersectingPeriods( period );
+
+ for ( final Integer organisationUnit : organisationUnits )
{
- final Collection<Integer> orgUnitChildren = hierarchy.getChildren( organisationUnit, group );
-
- double value = 0d;
+ final int unitLevel = operand.isHasAggregationLevels() ? aggregationCache.getLevelOfOrganisationUnit( organisationUnit ) : 0;
- for ( Integer orgUnitChild : orgUnitChildren )
+ for ( OrganisationUnitGroup group : organisationUnitGroups )
{
- final int dataValueLevel = operand.isHasAggregationLevels() ? aggregationCache.getLevelOfOrganisationUnit( orgUnitChild ) : 0;
-
- if ( operand.isHasAggregationLevels() && !operand.aggregationLevelIsValid( unitLevel, dataValueLevel ) )
- {
- continue;
- }
-
- final double[] entry = getAggregate( orgUnitChild, crossTabValues, period.getStartDate(), period.getEndDate(), unitLevel ); // <Org unit, [total value, total relevant days]>
-
- if ( !MathUtils.isZero( entry[1] ) )
- {
- double average = entry[ 0 ] / entry[ 1 ];
+ final Set<Integer> orgUnitChildren = orgUnitHierarchy.getChildren( organisationUnit, group );
+
+ aggregationCache.filterForAggregationLevel( orgUnitChildren, operand, unitLevel );
+
+ double value = 0d;
+
+ for ( Integer orgUnitChild : orgUnitChildren )
+ {
+ double totalValue = 0d;
+ double totalRelevantDays = 0d;
- value += average;
- }
- }
-
- if ( !MathUtils.isZero( value ) )
- {
- values.add( new OrgUnitOperand( organisationUnit, group != null ? group.getId() : 0, value ) );
+ for ( Integer intersectingPeriod : intersectingPeriods )
+ {
+ final String val = crossTabValues.get( intersectingPeriod + CrossTabStore.SEPARATOR + orgUnitChild );
+
+ if ( val != null )
+ {
+ final double[] entry = getAggregate( orgUnitChild, aggregationCache.getPeriod( intersectingPeriod ), val, period.getStartDate(), period.getEndDate(), unitLevel ); // <Org unit, [total value, total relevant days]>
+
+ totalValue += entry[0];
+ totalRelevantDays += entry[1];
+ }
+ }
+
+ if ( !MathUtils.isZero( totalRelevantDays ) )
+ {
+ double average = totalValue / totalRelevantDays;
+
+ value += average;
+ }
+ }
+
+ if ( !MathUtils.isZero( value ) )
+ {
+ values.add( new OrgUnitOperand( period.getId(), periodType.getId(), organisationUnit, group != null ? group.getId() : 0, value ) );
+ }
}
}
- }
+ }
return values;
}
- private double[] getAggregate( int organisationUnit, Collection<CrossTabDataValue> crossTabValues,
- Date startDate, Date endDate, int unitLevel )
+ private double[] getAggregate( int organisationUnit, Period period, String val, Date startDate, Date endDate, int unitLevel )
{
- double totalValue = 0d;
- double totalRelevantDays = 0d;
+ double value = 0.0;
+ double relevantDays = 0.0;
+
+ final Date currentStartDate = period.getStartDate();
+ final Date currentEndDate = period.getEndDate();
- for ( final CrossTabDataValue crossTabValue : crossTabValues )
+ final double duration = getDaysInclusive( currentStartDate, currentEndDate );
+
+ if ( duration > 0 )
{
- final Period period = aggregationCache.getPeriod( crossTabValue.getPeriodId() );
-
- final Date currentStartDate = period.getStartDate();
- final Date currentEndDate = period.getEndDate();
-
- final double duration = getDaysInclusive( currentStartDate, currentEndDate );
-
- if ( duration > 0 )
- {
- double value = 0.0;
- double relevantDays = 0.0;
-
- String val = crossTabValue.getValueMap().get( organisationUnit );
-
- if ( val == null )
- {
- continue;
- }
-
- try
- {
- value = Double.parseDouble( val );
- }
- catch ( NumberFormatException ex )
- {
- log.warn( "Value skipped, not numeric: '" + val );
- continue;
- }
-
- if ( currentStartDate.compareTo( startDate ) >= 0 && currentEndDate.compareTo( endDate ) <= 0 ) // Value is within period
- {
- relevantDays = getDaysInclusive( currentStartDate, currentEndDate );
- }
- else if ( currentStartDate.compareTo( startDate ) <= 0 && currentEndDate.compareTo( endDate ) >= 0 ) // Value spans whole period
- {
- relevantDays = getDaysInclusive( startDate, endDate );
- }
- else if ( currentStartDate.compareTo( startDate ) <= 0 && currentEndDate.compareTo( startDate ) >= 0
- && currentEndDate.compareTo( endDate ) <= 0 ) // Value spans period start
- {
- relevantDays = getDaysInclusive( startDate, currentEndDate );
- }
- else if ( currentStartDate.compareTo( startDate ) >= 0 && currentStartDate.compareTo( endDate ) <= 0
- && currentEndDate.compareTo( endDate ) >= 0 ) // Value spans period end
- {
- relevantDays = getDaysInclusive( currentStartDate, endDate );
- }
-
- value = value * relevantDays;
-
- totalValue += value;
- totalRelevantDays += relevantDays;
- }
- }
+ value = Double.parseDouble( val );
+
+ if ( currentStartDate.compareTo( startDate ) >= 0 && currentEndDate.compareTo( endDate ) <= 0 ) // Value is within period
+ {
+ relevantDays = getDaysInclusive( currentStartDate, currentEndDate );
+ }
+ else if ( currentStartDate.compareTo( startDate ) <= 0 && currentEndDate.compareTo( endDate ) >= 0 ) // Value spans whole period
+ {
+ relevantDays = getDaysInclusive( startDate, endDate );
+ }
+ else if ( currentStartDate.compareTo( startDate ) <= 0 && currentEndDate.compareTo( startDate ) >= 0
+ && currentEndDate.compareTo( endDate ) <= 0 ) // Value spans period start
+ {
+ relevantDays = getDaysInclusive( startDate, currentEndDate );
+ }
+ else if ( currentStartDate.compareTo( startDate ) >= 0 && currentStartDate.compareTo( endDate ) <= 0
+ && currentEndDate.compareTo( endDate ) >= 0 ) // Value spans period end
+ {
+ relevantDays = getDaysInclusive( currentStartDate, endDate );
+ }
+
+ value = value * relevantDays;
+ }
- final double[] values = { totalValue, totalRelevantDays };
+ final double[] values = { value, relevantDays };
return values;
}
- public boolean isApplicable( DataElementOperand operand, PeriodType periodType )
+ public boolean isApplicable( DataElementOperand operand )
{
- return ( operand.getValueType().equals( VALUE_TYPE_INT ) && operand.getAggregationOperator().equals( AGGREGATION_OPERATOR_AVERAGE ) &&
- operand.getFrequencyOrder() < periodType.getFrequencyOrder() );
- }
+ return operand.getValueType().equals( VALUE_TYPE_INT ) && operand.getAggregationOperator().equals( AGGREGATION_OPERATOR_AVERAGE );
+ }
}
=== removed file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntSingleValueAggregator.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntSingleValueAggregator.java 2012-08-25 11:47:17 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntSingleValueAggregator.java 1970-01-01 00:00:00 +0000
@@ -1,148 +0,0 @@
-package org.hisp.dhis.datamart.aggregation.dataelement;
-
-/*
- * Copyright (c) 2004-2012, University of Oslo
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- * * Neither the name of the HISP project nor the names of its contributors may
- * be used to endorse or promote products derived from this software without
- * specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
- * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
- * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-import static org.hisp.dhis.dataelement.DataElement.AGGREGATION_OPERATOR_AVERAGE;
-import static org.hisp.dhis.dataelement.DataElement.VALUE_TYPE_INT;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.hisp.dhis.dataelement.DataElementOperand;
-import org.hisp.dhis.datamart.CrossTabDataValue;
-import org.hisp.dhis.datamart.OrgUnitOperand;
-import org.hisp.dhis.datamart.aggregation.cache.AggregationCache;
-import org.hisp.dhis.datamart.crosstab.CrossTabService;
-import org.hisp.dhis.organisationunit.OrganisationUnitGroup;
-import org.hisp.dhis.organisationunit.OrganisationUnitHierarchy;
-import org.hisp.dhis.period.Period;
-import org.hisp.dhis.period.PeriodType;
-import org.hisp.dhis.system.util.MathUtils;
-
-/**
- * @author Lars Helge Overland
- */
-public class AverageIntSingleValueAggregator
- implements DataElementAggregator
-{
- private static final Log log = LogFactory.getLog( AverageIntSingleValueAggregator.class );
-
- // -------------------------------------------------------------------------
- // Dependencies
- // -------------------------------------------------------------------------
-
- private CrossTabService crossTabService;
-
- public void setCrossTabService( CrossTabService crossTabService )
- {
- this.crossTabService = crossTabService;
- }
-
- protected AggregationCache aggregationCache;
-
- public void setAggregationCache( AggregationCache aggregationCache )
- {
- this.aggregationCache = aggregationCache;
- }
-
- // -------------------------------------------------------------------------
- // DataElementAggregator implementation
- // -------------------------------------------------------------------------
-
- public List<OrgUnitOperand> getAggregatedValues( DataElementOperand operand, Period period,
- Collection<Integer> organisationUnits, Collection<OrganisationUnitGroup> organisationUnitGroups, OrganisationUnitHierarchy hierarchy, String key )
- {
- final Collection<CrossTabDataValue> crossTabValues = crossTabService.getCrossTabDataValues( operand,
- aggregationCache.getIntersectingPeriods( period.getStartDate(), period.getEndDate() ), hierarchy.getChildren( organisationUnits ), key );
-
- final List<OrgUnitOperand> values = new ArrayList<OrgUnitOperand>();
-
- if ( crossTabValues.size() == 0 )
- {
- return values;
- }
-
- for ( Integer organisationUnit : organisationUnits )
- {
- final int unitLevel = operand.isHasAggregationLevels() ? aggregationCache.getLevelOfOrganisationUnit( organisationUnit ) : 0; //TODO aggregation level
-
- for ( OrganisationUnitGroup group : organisationUnitGroups )
- {
- final Collection<Integer> orgUnitChildren = hierarchy.getChildren( organisationUnit, group ); // TODO group
-
- double value = 0d;
-
- for ( Integer orgUnitChild : orgUnitChildren )
- {
- final int dataValueLevel = operand.isHasAggregationLevels() ? aggregationCache.getLevelOfOrganisationUnit( orgUnitChild ) : 0; //TODO aggregation level
-
- if ( operand.isHasAggregationLevels() && !operand.aggregationLevelIsValid( unitLevel, dataValueLevel ) )
- {
- continue;
- }
-
- for ( CrossTabDataValue crossTabValue : crossTabValues ) // Iterate periods
- {
- String val = crossTabValue.getValueMap().get( orgUnitChild ); // TODO make set of periods, avoid iteration?
-
- if ( val == null )
- {
- continue;
- }
-
- try
- {
- value += Double.parseDouble( val );
- }
- catch ( NumberFormatException ex )
- {
- log.warn( "Value skipped, not numeric: '" + val );
- continue;
- }
- }
- }
-
- if ( !MathUtils.isZero( value ) )
- {
- values.add( new OrgUnitOperand( organisationUnit, group != null ? group.getId() : 0, value ) );
- }
- }
- }
-
- return values;
- }
-
- public boolean isApplicable( DataElementOperand operand, PeriodType periodType )
- {
- return ( operand.getValueType().equals( VALUE_TYPE_INT ) && operand.getAggregationOperator().equals( AGGREGATION_OPERATOR_AVERAGE ) &&
- operand.getFrequencyOrder() >= periodType.getFrequencyOrder() );
- }
-}
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/DataElementAggregator.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/DataElementAggregator.java 2012-08-25 11:47:17 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/DataElementAggregator.java 2012-08-29 14:39:44 +0000
@@ -37,7 +37,7 @@
import org.hisp.dhis.organisationunit.OrganisationUnitGroup;
import org.hisp.dhis.organisationunit.OrganisationUnitHierarchy;
import org.hisp.dhis.period.Period;
-import org.hisp.dhis.period.PeriodType;
+import org.hisp.dhis.period.PeriodHierarchy;
/**
* @author Lars Helge Overland
@@ -47,8 +47,9 @@
final String TRUE = "true";
final Map<DataElementOperand, Double> EMPTY_MAP = new HashMap<DataElementOperand, Double>();
- List<OrgUnitOperand> getAggregatedValues( DataElementOperand operand, Period period,
- Collection<Integer> organisationUnits, Collection<OrganisationUnitGroup> organisationUnitGroups, OrganisationUnitHierarchy hierarchy, String key );
+ List<OrgUnitOperand> getAggregatedValues( DataElementOperand operand, Collection<Period> periods,
+ Collection<Integer> organisationUnits, Collection<OrganisationUnitGroup> organisationUnitGroups,
+ PeriodHierarchy periodHierarchy, OrganisationUnitHierarchy hierarchy, String key );
- boolean isApplicable( DataElementOperand operand, PeriodType periodType );
+ boolean isApplicable( DataElementOperand operand );
}
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumBoolAggregator.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumBoolAggregator.java 2012-08-25 11:47:17 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumBoolAggregator.java 2012-08-29 14:39:44 +0000
@@ -33,15 +33,18 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.hisp.dhis.dataelement.DataElementOperand;
-import org.hisp.dhis.datamart.CrossTabDataValue;
import org.hisp.dhis.datamart.OrgUnitOperand;
import org.hisp.dhis.datamart.aggregation.cache.AggregationCache;
import org.hisp.dhis.datamart.crosstab.CrossTabService;
+import org.hisp.dhis.datamart.crosstab.jdbc.CrossTabStore;
import org.hisp.dhis.organisationunit.OrganisationUnitGroup;
import org.hisp.dhis.organisationunit.OrganisationUnitHierarchy;
import org.hisp.dhis.period.Period;
+import org.hisp.dhis.period.PeriodHierarchy;
import org.hisp.dhis.period.PeriodType;
import org.hisp.dhis.system.util.MathUtils;
@@ -73,11 +76,12 @@
// DataElementAggregator implementation
// -------------------------------------------------------------------------
- public List<OrgUnitOperand> getAggregatedValues( DataElementOperand operand, Period period,
- Collection<Integer> organisationUnits, Collection<OrganisationUnitGroup> organisationUnitGroups, OrganisationUnitHierarchy hierarchy, String key )
+ public List<OrgUnitOperand> getAggregatedValues( DataElementOperand operand, Collection<Period> periods,
+ Collection<Integer> organisationUnits, Collection<OrganisationUnitGroup> organisationUnitGroups,
+ PeriodHierarchy periodHierarchy, OrganisationUnitHierarchy orgUnitHierarchy, String key )
{
- final Collection<CrossTabDataValue> crossTabValues = crossTabService.getCrossTabDataValues( operand,
- aggregationCache.getPeriodsBetweenDates( period.getStartDate(), period.getEndDate() ), hierarchy.getChildren( organisationUnits ), key );
+ final Map<String, String> crossTabValues = crossTabService.getCrossTabDataValues( operand,
+ periodHierarchy.getPeriodsBetween( periods ), orgUnitHierarchy.getChildren( organisationUnits ), key );
final List<OrgUnitOperand> values = new ArrayList<OrgUnitOperand>();
@@ -85,45 +89,47 @@
{
return values;
}
-
- for ( Integer organisationUnit : organisationUnits )
+
+ for ( Period period : periods )
{
- final int unitLevel = operand.isHasAggregationLevels() ? aggregationCache.getLevelOfOrganisationUnit( organisationUnit ) : 0;
-
- for ( OrganisationUnitGroup group : organisationUnitGroups )
- {
- final Collection<Integer> orgUnitChildren = hierarchy.getChildren( organisationUnit, group );
-
- double value = 0d;
-
- for ( Integer orgUnitChild : orgUnitChildren )
- {
- final int dataValueLevel = operand.isHasAggregationLevels() ? aggregationCache.getLevelOfOrganisationUnit( orgUnitChild ) : 0;
-
- if ( operand.isHasAggregationLevels() && !operand.aggregationLevelIsValid( unitLevel, dataValueLevel ) )
- {
- continue;
- }
-
- for ( CrossTabDataValue crossTabValue : crossTabValues )
- {
- String val = crossTabValue.getValueMap().get( orgUnitChild );
-
- if ( val == null )
- {
- continue;
- }
-
- if ( val.toLowerCase().equals( TRUE ) )
- {
- value++;
- }
- }
- }
-
- if ( !MathUtils.isZero( value ) )
- {
- values.add( new OrgUnitOperand( organisationUnit, group != null ? group.getId() : 0, value ) );
+ final PeriodType periodType = period.getPeriodType();
+
+ if ( !isApplicable( operand, periodType ) )
+ {
+ continue;
+ }
+
+ final Collection<Integer> periodsBetween = periodHierarchy.getPeriodsBetween( period );
+
+ for ( Integer organisationUnit : organisationUnits )
+ {
+ final int unitLevel = operand.isHasAggregationLevels() ? aggregationCache.getLevelOfOrganisationUnit( organisationUnit ) : 0;
+
+ for ( OrganisationUnitGroup group : organisationUnitGroups )
+ {
+ final Set<Integer> orgUnitChildren = orgUnitHierarchy.getChildren( organisationUnit, group );
+
+ aggregationCache.filterForAggregationLevel( orgUnitChildren, operand, unitLevel );
+
+ double value = 0d;
+
+ for ( Integer periodBetween : periodsBetween )
+ {
+ for ( Integer orgUnitChild : orgUnitChildren )
+ {
+ final String val = crossTabValues.get( periodBetween + CrossTabStore.SEPARATOR + orgUnitChild );
+
+ if ( TRUE.equalsIgnoreCase( val ) )
+ {
+ value++;
+ }
+ }
+ }
+
+ if ( !MathUtils.isZero( value ) )
+ {
+ values.add( new OrgUnitOperand( period.getId(), periodType.getId(), organisationUnit, group != null ? group.getId() : 0, value ) );
+ }
}
}
}
@@ -131,9 +137,13 @@
return values;
}
+ public boolean isApplicable( DataElementOperand operand )
+ {
+ return operand.getValueType().equals( VALUE_TYPE_BOOL ) && operand.getAggregationOperator().equals( AGGREGATION_OPERATOR_SUM );
+ }
+
public boolean isApplicable( DataElementOperand operand, PeriodType periodType )
{
- return ( operand.getValueType().equals( VALUE_TYPE_BOOL ) && operand.getAggregationOperator().equals( AGGREGATION_OPERATOR_SUM ) &&
- operand.getFrequencyOrder() <= periodType.getFrequencyOrder() ); // Ignore disaggregation
+ return operand.getFrequencyOrder() <= periodType.getFrequencyOrder(); // Ignore disaggregation
}
}
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumIntAggregator.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumIntAggregator.java 2012-08-25 11:47:17 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumIntAggregator.java 2012-08-29 14:39:44 +0000
@@ -33,17 +33,18 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.hisp.dhis.dataelement.DataElementOperand;
-import org.hisp.dhis.datamart.CrossTabDataValue;
import org.hisp.dhis.datamart.OrgUnitOperand;
import org.hisp.dhis.datamart.aggregation.cache.AggregationCache;
import org.hisp.dhis.datamart.crosstab.CrossTabService;
+import org.hisp.dhis.datamart.crosstab.jdbc.CrossTabStore;
import org.hisp.dhis.organisationunit.OrganisationUnitGroup;
import org.hisp.dhis.organisationunit.OrganisationUnitHierarchy;
import org.hisp.dhis.period.Period;
+import org.hisp.dhis.period.PeriodHierarchy;
import org.hisp.dhis.period.PeriodType;
import org.hisp.dhis.system.util.MathUtils;
@@ -53,8 +54,6 @@
public class SumIntAggregator
implements DataElementAggregator
{
- private static final Log log = LogFactory.getLog( SumIntAggregator.class );
-
// -------------------------------------------------------------------------
// Dependencies
// -------------------------------------------------------------------------
@@ -77,11 +76,12 @@
// DataElementAggregator implementation
// -------------------------------------------------------------------------
- public List<OrgUnitOperand> getAggregatedValues( DataElementOperand operand, Period period,
- Collection<Integer> organisationUnits, Collection<OrganisationUnitGroup> organisationUnitGroups, OrganisationUnitHierarchy hierarchy, String key )
+ public List<OrgUnitOperand> getAggregatedValues( DataElementOperand operand, Collection<Period> periods,
+ Collection<Integer> organisationUnits, Collection<OrganisationUnitGroup> organisationUnitGroups,
+ PeriodHierarchy periodHierarchy, OrganisationUnitHierarchy orgUnitHierarchy, String key )
{
- final Collection<CrossTabDataValue> crossTabValues = crossTabService.getCrossTabDataValues( operand,
- aggregationCache.getPeriodsBetweenDates( period.getStartDate(), period.getEndDate() ), hierarchy.getChildren( organisationUnits ), key );
+ final Map<String, String> crossTabValues = crossTabService.getCrossTabDataValues( operand,
+ periodHierarchy.getPeriodsBetween( periods ), orgUnitHierarchy.getChildren( organisationUnits ), key );
final List<OrgUnitOperand> values = new ArrayList<OrgUnitOperand>();
@@ -90,49 +90,46 @@
return values;
}
- for ( Integer organisationUnit : organisationUnits )
+ for ( Period period : periods )
{
- final int unitLevel = operand.isHasAggregationLevels() ? aggregationCache.getLevelOfOrganisationUnit( organisationUnit ) : 0;
-
- for ( OrganisationUnitGroup group : organisationUnitGroups )
- {
- final Collection<Integer> orgUnitChildren = hierarchy.getChildren( organisationUnit, group );
-
- double value = 0d;
-
- for ( Integer orgUnitChild : orgUnitChildren )
- {
- final int dataValueLevel = operand.isHasAggregationLevels() ? aggregationCache.getLevelOfOrganisationUnit( orgUnitChild ) : 0;
-
- if ( operand.isHasAggregationLevels() && !operand.aggregationLevelIsValid( unitLevel, dataValueLevel ) )
- {
- continue;
- }
-
- for ( CrossTabDataValue crossTabValue : crossTabValues )
- {
- final String val = crossTabValue.getValueMap().get( orgUnitChild );
-
- if ( val == null )
- {
- continue;
- }
-
- try
- {
- value += Double.parseDouble( val );
- }
- catch ( NumberFormatException ex )
- {
- log.warn( "Value skipped, not numeric: '" + val );
- continue;
- }
- }
- }
-
- if ( !MathUtils.isZero( value ) )
- {
- values.add( new OrgUnitOperand( organisationUnit, group != null ? group.getId() : 0, value ) );
+ final PeriodType periodType = period.getPeriodType();
+
+ if ( !isApplicable( operand, periodType ) )
+ {
+ continue;
+ }
+
+ final Collection<Integer> periodsBetween = periodHierarchy.getPeriodsBetween( period );
+
+ for ( Integer organisationUnit : organisationUnits )
+ {
+ final int unitLevel = operand.isHasAggregationLevels() ? aggregationCache.getLevelOfOrganisationUnit( organisationUnit ) : 0;
+
+ for ( OrganisationUnitGroup group : organisationUnitGroups )
+ {
+ final Set<Integer> orgUnitChildren = orgUnitHierarchy.getChildren( organisationUnit, group );
+
+ aggregationCache.filterForAggregationLevel( orgUnitChildren, operand, unitLevel );
+
+ double value = 0d;
+
+ for ( Integer periodBetween : periodsBetween )
+ {
+ for ( Integer orgUnitChild : orgUnitChildren )
+ {
+ final String val = crossTabValues.get( periodBetween + CrossTabStore.SEPARATOR + orgUnitChild );
+
+ if ( val != null )
+ {
+ value += Double.parseDouble( val );
+ }
+ }
+ }
+
+ if ( !MathUtils.isZero( value ) )
+ {
+ values.add( new OrgUnitOperand( period.getId(), periodType.getId(), organisationUnit, group != null ? group.getId() : 0, value ) );
+ }
}
}
}
@@ -140,9 +137,13 @@
return values;
}
+ public boolean isApplicable( DataElementOperand operand )
+ {
+ return operand.getValueType().equals( VALUE_TYPE_INT ) && operand.getAggregationOperator().equals( AGGREGATION_OPERATOR_SUM );
+ }
+
public boolean isApplicable( DataElementOperand operand, PeriodType periodType )
{
- return ( operand.getValueType().equals( VALUE_TYPE_INT ) && operand.getAggregationOperator().equals( AGGREGATION_OPERATOR_SUM ) &&
- operand.getFrequencyOrder() <= periodType.getFrequencyOrder() ); // Ignore disaggregation
+ return operand.getFrequencyOrder() <= periodType.getFrequencyOrder(); // Ignore disaggregation
}
}
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java 2012-08-25 11:47:17 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java 2012-08-29 14:39:44 +0000
@@ -33,7 +33,6 @@
import java.util.concurrent.Future;
import org.hisp.dhis.dataelement.DataElementOperand;
-import org.hisp.dhis.datamart.CrossTabDataValue;
import org.hisp.dhis.organisationunit.OrganisationUnit;
import org.hisp.dhis.organisationunit.OrganisationUnitGroup;
import org.hisp.dhis.period.Period;
@@ -107,14 +106,15 @@
/**
* Gets all CrossTabDataValues for the given collection of period identifiers
- * and organisation unit identifiers.
+ * and organisation unit identifiers as a map. The map key is a
+ * concatenation of <period identifier>-<organisation unit identifier>.
*
* @param operand the data element operand.
* @param periodIds the period identifiers.
* @param organisationUnitIds the organisation unit identifiers.
* @return collection of CrossTabDataValues.
*/
- Collection<CrossTabDataValue> getCrossTabDataValues( DataElementOperand operand,
+ Map<String, String> getCrossTabDataValues( DataElementOperand operand,
Collection<Integer> periodIds, Collection<Integer> organisationUnitIds, String key );
/**
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java 2012-08-25 11:47:17 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java 2012-08-29 14:39:44 +0000
@@ -43,7 +43,6 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.hisp.dhis.dataelement.DataElementOperand;
-import org.hisp.dhis.datamart.CrossTabDataValue;
import org.hisp.dhis.datamart.DataMartManager;
import org.hisp.dhis.datamart.crosstab.jdbc.CrossTabStore;
import org.hisp.dhis.jdbc.batchhandler.GenericBatchHandler;
@@ -307,7 +306,7 @@
crossTabStore.dropAggregatedOrgUnitDataCache( key );
}
- public Collection<CrossTabDataValue> getCrossTabDataValues( DataElementOperand operand,
+ public Map<String, String> getCrossTabDataValues( DataElementOperand operand,
Collection<Integer> periodIds, Collection<Integer> organisationUnitIds, String key )
{
return crossTabStore.getCrossTabDataValues( operand, periodIds, organisationUnitIds, key );
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/CrossTabStore.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/CrossTabStore.java 2012-08-25 11:47:17 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/CrossTabStore.java 2012-08-29 14:39:44 +0000
@@ -32,7 +32,6 @@
import java.util.Map;
import org.hisp.dhis.dataelement.DataElementOperand;
-import org.hisp.dhis.datamart.CrossTabDataValue;
/**
* @author Lars Helge Overland
@@ -45,6 +44,7 @@
final String COLUMN_PREFIX = "col";
final String AGGREGATEDDATA_CACHE_PREFIX = "aggregateddata_cache_";
final String AGGREGATEDORGUNITDATA_CACHE_PREFIX = "aggregatedorgunitdata_cache_";
+ final String SEPARATOR = "-";
/**
* Creates a crosstab table where the first column is the period identifier,
@@ -97,14 +97,15 @@
/**
* Gets all CrossTabDataValues for the given collection of period identifiers
- * and organisation unit identifiers.
+ * and organisation unit identifiers as a map. The map key is a
+ * concatenation of <period identifier>-<organisation unit identifier>.
*
* @param operand the data element operand.
* @param periodIds the period identifiers.
* @param organisationUnitIds the organisation unit identifiers.
* @return collection of CrossTabDataValues.
*/
- Collection<CrossTabDataValue> getCrossTabDataValues( DataElementOperand operand,
+ Map<String, String> getCrossTabDataValues( DataElementOperand operand,
Collection<Integer> periodIds, Collection<Integer> organisationUnitIds, String key );
/**
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/JDBCCrossTabStore.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/JDBCCrossTabStore.java 2012-08-25 11:47:17 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/JDBCCrossTabStore.java 2012-08-29 14:39:44 +0000
@@ -31,7 +31,6 @@
import java.sql.ResultSet;
import java.sql.SQLException;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
@@ -40,7 +39,6 @@
import org.amplecode.quick.StatementHolder;
import org.amplecode.quick.StatementManager;
import org.hisp.dhis.dataelement.DataElementOperand;
-import org.hisp.dhis.datamart.CrossTabDataValue;
/**
* @author Lars Helge Overland
@@ -135,7 +133,7 @@
// CrossTabDataValue
// -------------------------------------------------------------------------
- public Collection<CrossTabDataValue> getCrossTabDataValues( DataElementOperand operand,
+ public Map<String, String> getCrossTabDataValues( DataElementOperand operand,
Collection<Integer> periodIds, Collection<Integer> organisationUnitIds, String key )
{
final StatementHolder holder = statementManager.getHolder();
@@ -161,7 +159,11 @@
holder.close();
}
}
-
+
+ // -------------------------------------------------------------------------
+ // AggregatedDataCacheValue
+ // -------------------------------------------------------------------------
+
public Map<DataElementOperand, Double> getAggregatedDataCacheValue( Collection<DataElementOperand> operands,
int periodId, int sourceId, String key )
{
@@ -218,28 +220,24 @@
// Supportive methods
// -------------------------------------------------------------------------
- private Collection<CrossTabDataValue> getCrossTabDataValues( ResultSet resultSet, Collection<Integer> organisationUnitIds )
+ private Map<String, String> getCrossTabDataValues( ResultSet resultSet, Collection<Integer> organisationUnitIds )
throws SQLException
{
- final List<CrossTabDataValue> values = new ArrayList<CrossTabDataValue>();
+ final Map<String, String> values = new HashMap<String, String>();
while ( resultSet.next() )
- {
- final CrossTabDataValue value = new CrossTabDataValue();
-
- value.setPeriodId( resultSet.getInt( 3 ) );
-
- for ( Integer id : organisationUnitIds )
+ {
+ int periodId = resultSet.getInt( 3 );
+
+ for ( Integer organisationUnitId : organisationUnitIds )
{
- final String columnValue = resultSet.getString( CrossTabStore.COLUMN_PREFIX + id );
+ final String columnValue = resultSet.getString( CrossTabStore.COLUMN_PREFIX + organisationUnitId );
if ( columnValue != null )
{
- value.getValueMap().put( id, columnValue );
+ values.put( periodId + CrossTabStore.SEPARATOR + organisationUnitId, columnValue );
}
}
-
- values.add( value );
}
return values;
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DataElementDataMart.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DataElementDataMart.java 2012-08-22 16:13:29 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DataElementDataMart.java 2012-08-29 14:39:44 +0000
@@ -37,6 +37,7 @@
import org.hisp.dhis.organisationunit.OrganisationUnitGroup;
import org.hisp.dhis.organisationunit.OrganisationUnitHierarchy;
import org.hisp.dhis.period.Period;
+import org.hisp.dhis.period.PeriodHierarchy;
/**
* @author Lars Helge Overland
@@ -45,5 +46,5 @@
{
Future<?> exportDataValues( Collection<DataElementOperand> operands, Collection<Period> periods,
Collection<OrganisationUnit> organisationUnits, Collection<OrganisationUnitGroup> organisationUnitGroups,
- OrganisationUnitHierarchy hierarchy, Class<? extends BatchHandler<AggregatedDataValue>> clazz, String key );
+ PeriodHierarchy periodHierarchy, OrganisationUnitHierarchy hierarchy, Class<? extends BatchHandler<AggregatedDataValue>> clazz, String key );
}
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DefaultDataElementDataMart.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DefaultDataElementDataMart.java 2012-08-25 11:47:17 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DefaultDataElementDataMart.java 2012-08-29 14:39:44 +0000
@@ -30,7 +30,9 @@
import static org.hisp.dhis.system.util.MathUtils.getRounded;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.Future;
@@ -49,9 +51,8 @@
import org.hisp.dhis.organisationunit.OrganisationUnitGroup;
import org.hisp.dhis.organisationunit.OrganisationUnitHierarchy;
import org.hisp.dhis.period.Period;
-import org.hisp.dhis.period.PeriodType;
+import org.hisp.dhis.period.PeriodHierarchy;
import org.hisp.dhis.system.util.ConversionUtils;
-import org.hisp.dhis.system.util.SystemUtils;
import org.springframework.scheduling.annotation.Async;
/**
@@ -103,7 +104,7 @@
@Async
public Future<?> exportDataValues( Collection<DataElementOperand> operands, Collection<Period> periods,
Collection<OrganisationUnit> organisationUnits, Collection<OrganisationUnitGroup> organisationUnitGroups,
- OrganisationUnitHierarchy hierarchy, Class<? extends BatchHandler<AggregatedDataValue>> clazz, String key )
+ PeriodHierarchy periodHierarchy, OrganisationUnitHierarchy orgUnitHierarchy, Class<? extends BatchHandler<AggregatedDataValue>> clazz, String key )
{
statementManager.initialise(); // Running in separate thread
@@ -113,47 +114,42 @@
final Collection<Integer> organisationUnitIds = ConversionUtils.getIdentifiers( OrganisationUnit.class, organisationUnits );
+ final Map<DataElementOperand, DataElementAggregator> operandAggregatorMap = getAggregatorMap( operands );
+
populateHasAggregationLevels( operands );
organisationUnitGroups = organisationUnitGroups != null ? organisationUnitGroups : DataMartEngine.DUMMY_ORG_UNIT_GROUPS;
- for ( final Period period : periods )
+ for ( final DataElementOperand dataElementOperand : operands )
{
- operands : for ( final DataElementOperand operand : operands )
- {
- final PeriodType periodType = period.getPeriodType();
-
- final DataElementAggregator aggregator = getAggregator( operand, periodType );
-
- if ( aggregator == null )
- {
- continue operands;
- }
-
- final List<OrgUnitOperand> values = aggregator.getAggregatedValues( operand, period, organisationUnitIds, organisationUnitGroups, hierarchy, key );
-
- for ( OrgUnitOperand orgUnitOperand : values )
- {
- final int level = aggregationCache.getLevelOfOrganisationUnit( orgUnitOperand.getOrgUnitId() );
-
- final double value = getRounded( orgUnitOperand.getValue(), DECIMALS );
-
- aggregatedValue.clear();
-
- aggregatedValue.setDataElementId( operand.getDataElementId() );
- aggregatedValue.setCategoryOptionComboId( operand.getOptionComboId() );
- aggregatedValue.setPeriodId( period.getId() );
- aggregatedValue.setPeriodTypeId( period.getPeriodType().getId() );
- aggregatedValue.setOrganisationUnitId( orgUnitOperand.getOrgUnitId() );
- aggregatedValue.setOrganisationUnitGroupId( orgUnitOperand.getOrgUnitGroupId() );
- aggregatedValue.setLevel( level );
- aggregatedValue.setValue( value );
-
- batchHandler.addObject( aggregatedValue );
- }
- }
-
- log.debug( "Exported data values for period: " + period + ", " + SystemUtils.getMemoryString() );
+ final DataElementAggregator aggregator = operandAggregatorMap.get( dataElementOperand );
+
+ if ( aggregator == null )
+ {
+ continue;
+ }
+
+ final List<OrgUnitOperand> values = aggregator.getAggregatedValues( dataElementOperand, periods, organisationUnitIds, organisationUnitGroups, periodHierarchy, orgUnitHierarchy, key );
+
+ for ( OrgUnitOperand orgUnitOperand : values )
+ {
+ final int level = aggregationCache.getLevelOfOrganisationUnit( orgUnitOperand.getOrgUnitId() );
+
+ final double value = getRounded( orgUnitOperand.getValue(), DECIMALS );
+
+ aggregatedValue.clear();
+
+ aggregatedValue.setDataElementId( dataElementOperand.getDataElementId() );
+ aggregatedValue.setCategoryOptionComboId( dataElementOperand.getOptionComboId() );
+ aggregatedValue.setPeriodId( orgUnitOperand.getPeriodId() );
+ aggregatedValue.setPeriodTypeId( orgUnitOperand.getPeriodTypeId() );
+ aggregatedValue.setOrganisationUnitId( orgUnitOperand.getOrgUnitId() );
+ aggregatedValue.setOrganisationUnitGroupId( orgUnitOperand.getOrgUnitGroupId() );
+ aggregatedValue.setLevel( level );
+ aggregatedValue.setValue( value );
+
+ batchHandler.addObject( aggregatedValue );
+ }
}
batchHandler.flush();
@@ -167,20 +163,26 @@
return null;
}
- private DataElementAggregator getAggregator( DataElementOperand operand, PeriodType periodType )
+ private Map<DataElementOperand, DataElementAggregator> getAggregatorMap( Collection<DataElementOperand> operands )
{
- for ( DataElementAggregator aggregator : aggregators )
+ Map<DataElementOperand, DataElementAggregator> map = new HashMap<DataElementOperand, DataElementAggregator>();
+
+ operand : for ( DataElementOperand operand : operands )
{
- if ( aggregator.isApplicable( operand, periodType ) )
+ for ( DataElementAggregator aggregator : aggregators )
{
- return aggregator;
+ if ( aggregator.isApplicable( operand ) )
+ {
+ map.put( operand, aggregator );
+ continue operand;
+ }
}
}
- return null;
+ return map;
}
- public void populateHasAggregationLevels( Collection<DataElementOperand> operands ) //TODO check
+ public void populateHasAggregationLevels( Collection<DataElementOperand> operands ) //TODO check effect
{
for ( DataElementOperand operand : operands )
{
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2012-08-25 11:47:17 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2012-08-29 14:39:44 +0000
@@ -62,6 +62,7 @@
import org.hisp.dhis.organisationunit.OrganisationUnitHierarchy;
import org.hisp.dhis.organisationunit.OrganisationUnitService;
import org.hisp.dhis.period.Period;
+import org.hisp.dhis.period.PeriodHierarchy;
import org.hisp.dhis.period.PeriodService;
import org.hisp.dhis.scheduling.TaskId;
import org.hisp.dhis.setting.SystemSettingManager;
@@ -299,6 +300,7 @@
//TODO improve index on datavalue for crosstab
final Collection<Integer> intersectingPeriodIds = ConversionUtils.getIdentifiers( Period.class, periodService.getIntersectionPeriods( periods ) );
+ final PeriodHierarchy periodHierarchy = periodService.getPeriodHierarchy( periods );
final Set<Integer> orgUnitChildrenIds = organisationUnitService.getOrganisationUnitHierarchy().getChildren( organisationUnitIds );
final List<Integer> crossTabOrgUnitIds = new ArrayList<Integer>( orgUnitChildrenIds );
@@ -337,14 +339,14 @@
if ( operandList.size() > 0 )
{
- final OrganisationUnitHierarchy hierarchy = organisationUnitService.getOrganisationUnitHierarchy().prepareChildren( organisationUnits );
+ final OrganisationUnitHierarchy orgUnitHierarchy = organisationUnitService.getOrganisationUnitHierarchy().prepareChildren( organisationUnits );
List<Future<?>> futures = new ArrayList<Future<?>>();
for ( List<DataElementOperand> operandPage : operandPages )
{
futures.add( dataElementDataMart.exportDataValues( operandPage, periods, organisationUnits,
- null, hierarchy, AggregatedDataValueTempBatchHandler.class, key ) );
+ null, periodHierarchy, orgUnitHierarchy, AggregatedDataValueTempBatchHandler.class, key ) );
}
ConcurrentUtils.waitForCompletion( futures );
@@ -403,14 +405,14 @@
if ( operandList.size() > 0 )
{
- final OrganisationUnitHierarchy hierarchy = organisationUnitService.getOrganisationUnitHierarchy().prepareChildren( groupOrganisationUnits, organisationUnitGroups );
+ final OrganisationUnitHierarchy orgUnitHierarchy = organisationUnitService.getOrganisationUnitHierarchy().prepareChildren( groupOrganisationUnits, organisationUnitGroups );
List<Future<?>> futures = new ArrayList<Future<?>>();
for ( List<DataElementOperand> operandPage : operandPages )
{
futures.add( dataElementDataMart.exportDataValues( operandPage, periods, groupOrganisationUnits,
- organisationUnitGroups, hierarchy, AggregatedOrgUnitDataValueTempBatchHandler.class, key ) );
+ organisationUnitGroups, periodHierarchy, orgUnitHierarchy, AggregatedOrgUnitDataValueTempBatchHandler.class, key ) );
}
ConcurrentUtils.waitForCompletion( futures );
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml 2012-08-25 11:47:17 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml 2012-08-29 14:39:44 +0000
@@ -67,11 +67,6 @@
<property name="aggregationCache" ref="org.hisp.dhis.datamart.aggregation.cache.AggregationCache" />
</bean>
- <bean id="org.hisp.dhis.datamart.aggregation.dataelement.AverageIntSingleValueAggregator" class="org.hisp.dhis.datamart.aggregation.dataelement.AverageIntSingleValueAggregator">
- <property name="crossTabService" ref="org.hisp.dhis.datamart.crosstab.CrossTabService" />
- <property name="aggregationCache" ref="org.hisp.dhis.datamart.aggregation.cache.AggregationCache" />
- </bean>
-
<bean id="org.hisp.dhis.datamart.aggregation.dataelement.AverageBoolAggregator" class="org.hisp.dhis.datamart.aggregation.dataelement.AverageBoolAggregator">
<property name="crossTabService" ref="org.hisp.dhis.datamart.crosstab.CrossTabService" />
<property name="aggregationCache" ref="org.hisp.dhis.datamart.aggregation.cache.AggregationCache" />
@@ -86,7 +81,6 @@
<property name="aggregators">
<list>
<ref bean="org.hisp.dhis.datamart.aggregation.dataelement.SumIntAggregator"/>
- <ref bean="org.hisp.dhis.datamart.aggregation.dataelement.AverageIntSingleValueAggregator"/>
<ref bean="org.hisp.dhis.datamart.aggregation.dataelement.AverageIntAggregator"/>
<ref bean="org.hisp.dhis.datamart.aggregation.dataelement.SumBoolAggregator"/>
<ref bean="org.hisp.dhis.datamart.aggregation.dataelement.AverageBoolAggregator"/>
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/test/java/org/hisp/dhis/datamart/DataMartServiceTest.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/test/java/org/hisp/dhis/datamart/DataMartServiceTest.java 2012-08-25 11:47:17 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/test/java/org/hisp/dhis/datamart/DataMartServiceTest.java 2012-08-29 14:39:44 +0000
@@ -362,13 +362,13 @@
assertEquals( aggregatedDataValueService.getAggregatedValue( dataElementB, categoryOptionCombo, periodA, unitG ), 0.0 );
assertEquals( aggregatedDataValueService.getAggregatedValue( dataElementB, categoryOptionCombo, periodA, unitH ), 100.0 );
- assertEquals( aggregatedDataValueService.getAggregatedValue( dataElementB, categoryOptionCombo, periodD, unitA ), 66.8, 0.5 );
- assertEquals( aggregatedDataValueService.getAggregatedValue( dataElementB, categoryOptionCombo, periodD, unitB ), 66.8, 0.5 );
- assertEquals( aggregatedDataValueService.getAggregatedValue( dataElementB, categoryOptionCombo, periodD, unitC ), 65.9, 0.5 );
- assertEquals( aggregatedDataValueService.getAggregatedValue( dataElementB, categoryOptionCombo, periodD, unitD ), 67.0, 0.5 );
- assertEquals( aggregatedDataValueService.getAggregatedValue( dataElementB, categoryOptionCombo, periodD, unitE ), 33.0, 0.5 );
+ assertEquals( aggregatedDataValueService.getAggregatedValue( dataElementB, categoryOptionCombo, periodD, unitA ), 66.7, 0.5 );
+ assertEquals( aggregatedDataValueService.getAggregatedValue( dataElementB, categoryOptionCombo, periodD, unitB ), 66.7, 0.5 );
+ assertEquals( aggregatedDataValueService.getAggregatedValue( dataElementB, categoryOptionCombo, periodD, unitC ), 66.7, 0.5 );
+ assertEquals( aggregatedDataValueService.getAggregatedValue( dataElementB, categoryOptionCombo, periodD, unitD ), 66.7, 0.5 );
+ assertEquals( aggregatedDataValueService.getAggregatedValue( dataElementB, categoryOptionCombo, periodD, unitE ), 33.3, 0.5 );
assertEquals( aggregatedDataValueService.getAggregatedValue( dataElementB, categoryOptionCombo, periodD, unitF ), 78.0, 0.5 );
- assertEquals( aggregatedDataValueService.getAggregatedValue( dataElementB, categoryOptionCombo, periodD, unitG ), 34.1, 0.5 );
+ assertEquals( aggregatedDataValueService.getAggregatedValue( dataElementB, categoryOptionCombo, periodD, unitG ), 33.1, 0.5 );
assertEquals( aggregatedDataValueService.getAggregatedValue( dataElementB, categoryOptionCombo, periodD, unitH ), 100.0, 0.5 );
}
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/test/java/org/hisp/dhis/datamart/crosstab/CrossTabServiceTest.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/test/java/org/hisp/dhis/datamart/crosstab/CrossTabServiceTest.java 2012-08-25 11:47:17 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/test/java/org/hisp/dhis/datamart/crosstab/CrossTabServiceTest.java 2012-08-29 14:39:44 +0000
@@ -29,7 +29,6 @@
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertNotNull;
-import static junit.framework.Assert.assertTrue;
import java.util.ArrayList;
import java.util.Calendar;
@@ -51,7 +50,6 @@
import org.hisp.dhis.dataelement.DataElementCategoryService;
import org.hisp.dhis.dataelement.DataElementOperand;
import org.hisp.dhis.dataelement.DataElementService;
-import org.hisp.dhis.datamart.CrossTabDataValue;
import org.hisp.dhis.datamart.crosstab.jdbc.CrossTabStore;
import org.hisp.dhis.datavalue.DataValueService;
import org.hisp.dhis.jdbc.batchhandler.GenericBatchHandler;
@@ -196,23 +194,15 @@
for ( DataElementOperand operand : operands )
{
- Collection<CrossTabDataValue> values = crossTabService.getCrossTabDataValues( operand, periodIds, organisationUnitIds, key );
+ Map<String, String> values = crossTabService.getCrossTabDataValues( operand, periodIds, organisationUnitIds, key );
assertNotNull( values );
- assertEquals( 5, values.size() );
+ assertEquals( 25, values.size() );
- for ( CrossTabDataValue crossTabValue : values )
+ for ( String valueKey : values.keySet() )
{
- assertTrue( crossTabValue.getPeriodId() != 0 );
-
- assertNotNull( crossTabValue.getValueMap() );
- assertEquals( 5, crossTabValue.getValueMap().size() );
-
- for ( String value : crossTabValue.getValueMap().values() )
- {
- assertEquals( "10", value );
- }
+ assertEquals( "10", values.get( valueKey ) );
}
}
}
