dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #20546
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 9415: Data mart. Improved performance on aggregation of average data elements and changed behavior slig...
------------------------------------------------------------
revno: 9415
committer: Lars Helge Øverland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Sun 2012-12-30 11:52:46 +0100
message:
Data mart. Improved performance on aggregation of average data elements and changed behavior slightly, the number of days in the aggregation period is used as denominator instead of number of days where there are data present.
modified:
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageBoolAggregator.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntAggregator.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/CrossTabStore.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/JDBCCrossTabStore.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageBoolAggregator.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageBoolAggregator.java 2012-11-08 08:33:41 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageBoolAggregator.java 2012-12-30 10:52:46 +0000
@@ -86,9 +86,9 @@
aggregationCache.getIntersectingPeriods( period.getStartDate(), period.getEndDate() ), organisationUnits, key );
final Map<DataElementOperand, double[]> entries = getAggregate( crossTabValues, period.getStartDate(),
- period.getEndDate(), period.getStartDate(), period.getEndDate(), unitLevel ); // <Operand, [total value, total relevant days]>
+ period.getEndDate(), period.getStartDate(), period.getEndDate(), unitLevel ); // <Operand, [total value, total days]>
- final Map<DataElementOperand, Double> values = new HashMap<DataElementOperand, Double>(); // <Operand, total value>
+ final Map<DataElementOperand, Double> values = new HashMap<DataElementOperand, Double>(); // <Operand, average value>
double average = 0.0;
@@ -108,8 +108,8 @@
private Map<DataElementOperand, double[]> getAggregate( final Collection<CrossTabDataValue> crossTabValues,
final Date startDate, final Date endDate, final Date aggregationStartDate, final Date aggregationEndDate, int unitLevel )
{
- final Map<DataElementOperand, double[]> totalSums = new HashMap<DataElementOperand, double[]>(); // <Operand, [total value, total relevant days]>
-
+ final Map<DataElementOperand, double[]> totalSums = new HashMap<DataElementOperand, double[]>(); // <Operand, [total value, total days]>
+
for ( final CrossTabDataValue crossTabValue : crossTabValues )
{
final Period period = aggregationCache.getPeriod( crossTabValue.getPeriodId() );
@@ -156,9 +156,9 @@
final double[] totalSum = totalSums.get( entry.getKey() );
value += totalSum != null ? totalSum[0] : 0;
- relevantDays += totalSum != null ? totalSum[1] : 0;
+ final double days = totalSum != null ? totalSum[1] + duration : duration;
- final double[] values = { value, relevantDays };
+ final double[] values = { value, days };
totalSums.put( entry.getKey(), values );
}
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntAggregator.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntAggregator.java 2012-11-08 08:33:41 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntAggregator.java 2012-12-30 10:52:46 +0000
@@ -86,37 +86,28 @@
return EMPTY_MAP;
}
- final Map<DataElementOperand, Double> values = new HashMap<DataElementOperand, Double>(); // <Operand, total value>
-
- for ( final Integer unitId : organisationUnits )
+ double days = getDaysInclusive( period.getStartDate(), period.getEndDate() );
+
+ final Collection<CrossTabDataValue> crossTabValues = crossTabService.getCrossTabDataValues( operands,
+ aggregationCache.getIntersectingPeriods( period.getStartDate(), period.getEndDate() ), organisationUnits, key );
+
+ final Map<DataElementOperand, Double> entries = getAggregate( crossTabValues, period.getStartDate(),
+ period.getEndDate(), unitLevel ); // <Operand, days x value>
+
+ for ( DataElementOperand operand : entries.keySet() )
{
- final Collection<CrossTabDataValue> crossTabValues =
- crossTabService.getCrossTabDataValues( operands, aggregationCache.getIntersectingPeriods( period.getStartDate(), period.getEndDate() ), unitId, key );
-
- final Map<DataElementOperand, double[]> entries = getAggregate( crossTabValues, period.getStartDate(),
- period.getEndDate(), period.getStartDate(), period.getEndDate(), unitLevel ); // <Operand, [total value, total relevant days]>
-
- for ( final Entry<DataElementOperand, double[]> entry : entries.entrySet() )
- {
- if ( entry.getValue() != null && entry.getValue()[ 1 ] > 0 )
- {
- double average = entry.getValue()[ 0 ] / entry.getValue()[ 1 ];
-
- average += values.containsKey( entry.getKey() ) ? values.get( entry.getKey() ) : 0;
-
- values.put( entry.getKey(), average );
- }
- }
- }
+ double value = entries.get( operand ) / days;
+ entries.put( operand, value );
+ }
- return values;
+ return entries;
}
- private Map<DataElementOperand, double[]> getAggregate( final Collection<CrossTabDataValue> crossTabValues,
- final Date startDate, final Date endDate, final Date aggregationStartDate, final Date aggregationEndDate, int unitLevel )
+ private Map<DataElementOperand, Double> getAggregate( final Collection<CrossTabDataValue> crossTabValues,
+ final Date startDate, final Date endDate, int unitLevel )
{
- final Map<DataElementOperand, double[]> totalSums = new HashMap<DataElementOperand, double[]>(); // <Operand, [total value, total relevant days]>
-
+ final Map<DataElementOperand, Double> values = new HashMap<DataElementOperand, Double>(); // <Operand, [total value, total relevant days]>
+
for ( final CrossTabDataValue crossTabValue : crossTabValues )
{
final Period period = aggregationCache.getPeriod( crossTabValue.getPeriodId() );
@@ -126,13 +117,11 @@
final int dataValueLevel = aggregationCache.getLevelOfOrganisationUnit( crossTabValue.getSourceId() );
- final double duration = getDaysInclusive( currentStartDate, currentEndDate );
-
- if ( duration > 0 )
+ if ( getDaysInclusive( currentStartDate, currentEndDate ) > 0 )
{
for ( final Entry<DataElementOperand, String> entry : crossTabValue.getValueMap().entrySet() ) // <Operand, value>
{
- if ( entry.getValue() != null && entry.getKey().aggregationLevelIsValid( unitLevel, dataValueLevel ) )
+ if ( entry.getKey() != null && entry.getValue() != null && entry.getKey().aggregationLevelIsValid( unitLevel, dataValueLevel ) )
{
double value = 0.0;
double relevantDays = 0.0;
@@ -168,19 +157,15 @@
value = value * relevantDays;
- final double[] totalSum = totalSums.get( entry.getKey() );
- value += totalSum != null ? totalSum[0] : 0;
- relevantDays += totalSum != null ? totalSum[1] : 0;
-
- final double[] values = { value, relevantDays };
-
- totalSums.put( entry.getKey(), values );
+ final Double current = values.get( entry.getKey() );
+ value += current != null ? current : 0.0;
+ values.put( entry.getKey(), value );
}
}
}
}
- return totalSums;
+ return values;
}
public Collection<DataElementOperand> filterOperands( final Collection<DataElementOperand> operands, final PeriodType periodType )
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java 2012-11-08 08:33:41 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java 2012-12-30 10:52:46 +0000
@@ -110,17 +110,6 @@
Collection<Integer> sourceIds, String key );
/**
- * Gets all CrossTabDataValues for the given collection of period ids and the source id.
- *
- * @param dataElementIds the data element identifiers.
- * @param periodIds the period identifiers.
- * @param sourceId the source identifier.
- * @return collection of CrossTabDataValues.
- */
- Collection<CrossTabDataValue> getCrossTabDataValues( Collection<DataElementOperand> operands, Collection<Integer> periodIds,
- int sourceId, String key );
-
- /**
* Gets a map of DataElementOperands and corresponding Double aggregated data
* element value from the cache table. If the group argument is not null it
* will read from the aggregated org unit data cache, if null it will read from
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java 2012-11-08 08:33:41 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java 2012-12-30 10:52:46 +0000
@@ -212,12 +212,6 @@
return crossTabStore.getCrossTabDataValues( operands, periodIds, sourceIds, key );
}
- public Collection<CrossTabDataValue> getCrossTabDataValues( Collection<DataElementOperand> operands,
- Collection<Integer> periodIds, int sourceId, String key )
- {
- return crossTabStore.getCrossTabDataValues( operands, periodIds, sourceId, key );
- }
-
public Map<DataElementOperand, Double> getAggregatedDataCacheValue( Collection<DataElementOperand> operands,
Period period, OrganisationUnit unit, OrganisationUnitGroup group, String key )
{
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/CrossTabStore.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/CrossTabStore.java 2012-11-08 08:33:41 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/CrossTabStore.java 2012-12-30 10:52:46 +0000
@@ -106,17 +106,6 @@
Collection<Integer> sourceIds, String key );
/**
- * Gets all CrossTabDataValues for the given collection of period ids and the source id.
- *
- * @param dataElementIds the data element identifiers.
- * @param periodIds the period identifiers.
- * @param sourceId the source identifier.
- * @return collection of CrossTabDataValues.
- */
- Collection<CrossTabDataValue> getCrossTabDataValues( Collection<DataElementOperand> operands, Collection<Integer> periodIds,
- int sourceId, String key );
-
- /**
* Gets a map of DataElementOperands and corresponding Double aggregated data
* element value from the cache table.
*
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/JDBCCrossTabStore.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/JDBCCrossTabStore.java 2012-11-08 08:33:41 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/JDBCCrossTabStore.java 2012-12-30 10:52:46 +0000
@@ -139,9 +139,13 @@
{
final StatementHolder holder = statementManager.getHolder();
- final String operandIds = getCommadelimitedString( operands );
+ if ( operands.isEmpty() || periodIds.isEmpty() || sourceIds.isEmpty() )
+ {
+ return new ArrayList<CrossTabDataValue>();
+ }
- final String sql = "SELECT periodid, sourceid, " + operandIds + " FROM " + CROSSTAB_TABLE_PREFIX + key + " AS c WHERE c.periodid IN (" +
+ final String sql = "SELECT periodid, sourceid, " + getCommadelimitedString( operands ) +
+ " FROM " + CROSSTAB_TABLE_PREFIX + key + " AS c WHERE c.periodid IN (" +
getCommaDelimitedString( periodIds ) + ") AND c.sourceid IN (" + getCommaDelimitedString( sourceIds ) + ")";
try
@@ -159,33 +163,7 @@
holder.close();
}
}
-
- public Collection<CrossTabDataValue> getCrossTabDataValues( Collection<DataElementOperand> operands,
- Collection<Integer> periodIds, int sourceId, String key )
- {
- final StatementHolder holder = statementManager.getHolder();
-
- final String operandIds = getCommadelimitedString( operands );
- final String sql = "SELECT periodid, sourceid, " + operandIds + " FROM " + CROSSTAB_TABLE_PREFIX + key + " AS c WHERE c.periodid IN (" +
- getCommaDelimitedString( periodIds ) + ") AND c.sourceid = " + sourceId;
-
- try
- {
- final ResultSet resultSet = holder.getStatement().executeQuery( sql );
-
- return getCrossTabDataValues( resultSet, operands );
- }
- catch ( SQLException ex )
- {
- throw new RuntimeException( "Failed to get CrossTabDataValues", ex );
- }
- finally
- {
- holder.close();
- }
- }
-
public Map<DataElementOperand, Double> getAggregatedDataCacheValue( Collection<DataElementOperand> operands,
int periodId, int sourceId, String key )
{
