dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #20547
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 9416: Analytics, impl support for disaggregation for data elements with average aggregation operator
------------------------------------------------------------
revno: 9416
committer: Lars Helge Øverland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Sun 2012-12-30 17:23:41 +0100
message:
Analytics, impl support for disaggregation for data elements with average aggregation operator
modified:
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsManager.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java
dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java
dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/TextUtils.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsManager.java 2012-12-18 16:01:44 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsManager.java 2012-12-30 16:23:41 +0000
@@ -30,9 +30,30 @@
import java.util.Map;
import java.util.concurrent.Future;
+import org.hisp.dhis.system.util.ListMap;
+
public interface AnalyticsManager
{
- static final String SEP = "-";
+ static final char SEP = '-';
- Future<Map<String, Double>> getAggregatedDataValues( DataQueryParams params );
+ /**
+ * Retrieves aggregated data values for the given query. The data is returned
+ * as a mapping where the key is concatenated from the dimension options for
+ * all dimensions, and the value is the data value.
+ *
+ * @param params the query to retrieve aggregated data for.
+ * @return a map.
+ */
+ Future<Map<String, Double>> getAggregatedDataValues( DataQueryParams params );
+
+ /**
+ * Inserts entries for the aggregation periods mapped to each data period
+ * in the given data value map. Removes the original entry for the data period.
+ *
+ * @param dataValueMap map with entries for all data values produced for the query.
+ * @param params the query.
+ * @param dataPeriodAggregationPeriodMap the mapping between data periods and
+ * aggregation periods for this query.
+ */
+ void replaceDataPeriodsWithAggregationPeriods( Map<String, Double> dataValueMap, DataQueryParams params, ListMap<String, String> dataPeriodAggregationPeriodMap );
}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java 2012-12-27 19:15:15 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java 2012-12-30 16:23:41 +0000
@@ -27,6 +27,8 @@
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
+import static org.hisp.dhis.analytics.AggregationType.AVERAGE_DISAGGREGATION;
+
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -34,7 +36,10 @@
import java.util.Map;
import org.hisp.dhis.common.Dxf2Namespace;
+import org.hisp.dhis.period.Period;
+import org.hisp.dhis.period.PeriodType;
import org.hisp.dhis.system.util.CollectionUtils;
+import org.hisp.dhis.system.util.ListMap;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
@@ -67,7 +72,9 @@
private transient int organisationUnitLevel;
- private AggregationType aggregationType;
+ private transient AggregationType aggregationType;
+
+ private transient PeriodType dataPeriodType;
// -------------------------------------------------------------------------
// Constructors
@@ -94,17 +101,23 @@
this.periodType = params.getPeriodType();
this.organisationUnitLevel = params.getOrganisationUnitLevel();
this.aggregationType = params.getAggregationType();
+ this.dataPeriodType = params.getDataPeriodType();
}
// -------------------------------------------------------------------------
// Logic
// -------------------------------------------------------------------------
+ /**
+ * Creates a list of the names of all dimensions for this query. If the period
+ * type property is set, the period dimension name will be replaced by the name
+ * of the period type, if present. If the organisation unit level property
+ * is set, the organisation unit dimension name will be replaced by the name
+ * of the organisation unit level column.
+ */
public List<String> getDimensionNames()
{
- List<String> list = new ArrayList<String>();
-
- list.addAll( dimensions.keySet() );
+ List<String> list = getDimensionNamesAsList();
if ( categories )
{
@@ -124,11 +137,28 @@
return list;
}
+ /**
+ * Returns the index of the period dimension in the index list.
+ */
+ public int getPeriodDimensionIndex()
+ {
+ return getDimensionNamesAsList().indexOf( PERIOD_DIM_ID );
+ }
+
+ /**
+ * Returns a list of the names of all filters.
+ */
public List<String> getFilterNames()
{
return new ArrayList<String>( filters.keySet() );
}
-
+
+ /**
+ * Returns a mapping between the dimension names and dimension values. Inserts
+ * keys and values for the current period type column name and organisation
+ * unit level name, if the period type property and organisation unit level
+ * property are set.
+ */
public Map<String, List<String>> getDimensionMap()
{
Map<String, List<String>> map = new HashMap<String, List<String>>();
@@ -173,7 +203,62 @@
{
return this.aggregationType != null && this.aggregationType.equals( aggregationType );
}
-
+
+ /**
+ * Creates a mapping between the data periods, based on the data period type
+ * for this query, and the aggregation periods for this query.
+ */
+ public ListMap<String, String> getDataPeriodAggregationPeriodMap()
+ {
+ ListMap<String, String> map = new ListMap<String, String>();
+
+ if ( dataPeriodType != null )
+ {
+ for ( String period : this.getPeriods() )
+ {
+ Period aggregatePeriod = PeriodType.getPeriodFromIsoString( period );
+
+ Period dataPeriod = dataPeriodType.createPeriod( aggregatePeriod.getStartDate() );
+
+ map.putValue( dataPeriod.getIsoDate(), period );
+ }
+ }
+
+ return map;
+ }
+
+ /**
+ * Replaces the periods of this query with the corresponding data periods.
+ * Sets the period type to the data period type. This method is relevant only
+ * when then the data period type has lower frequency than the aggregation
+ * period type.
+ */
+ public void replaceAggregationPeriodsWithDataPeriods( ListMap<String, String> dataPeriodAggregationPeriodMap )
+ {
+ if ( isAggregationType( AVERAGE_DISAGGREGATION ) && dataPeriodType != null )
+ {
+ this.periodType = this.dataPeriodType.getName();
+
+ setPeriods( new ArrayList<String>( getDataPeriodAggregationPeriodMap().keySet() ) );
+ }
+ }
+
+ // -------------------------------------------------------------------------
+ // Logic
+ // -------------------------------------------------------------------------
+
+ /**
+ * Returns the dimension names as a list.
+ */
+ private List<String> getDimensionNamesAsList()
+ {
+ return new ArrayList<String>( dimensions.keySet() );
+ }
+
+ // -------------------------------------------------------------------------
+ // hashCode, equals and toString
+ // -------------------------------------------------------------------------
+
@Override
public int hashCode()
{
@@ -242,7 +327,7 @@
{
return "[Dimensions: " + dimensions + ", Filters: " + filters + "]";
}
-
+
// -------------------------------------------------------------------------
// Get and set methods for serialize properties
// -------------------------------------------------------------------------
@@ -412,4 +497,14 @@
{
this.aggregationType = aggregationType;
}
+
+ public PeriodType getDataPeriodType()
+ {
+ return dataPeriodType;
+ }
+
+ public void setDataPeriodType( PeriodType dataPeriodType )
+ {
+ this.dataPeriodType = dataPeriodType;
+ }
}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java 2012-12-27 19:15:15 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java 2012-12-30 16:23:41 +0000
@@ -45,6 +45,8 @@
import org.hisp.dhis.system.util.Timer;
import org.springframework.beans.factory.annotation.Autowired;
+import static org.hisp.dhis.analytics.AnalyticsManager.SEP;
+
public class DefaultAnalyticsService
implements AnalyticsService
{
@@ -76,7 +78,7 @@
for ( Map.Entry<String, Double> entry : map.entrySet() )
{
grid.addRow();
- grid.addValues( entry.getKey().split( AnalyticsManager.SEP ) );
+ grid.addValues( entry.getKey().split( String.valueOf( SEP ) ) );
grid.addValue( entry.getValue() );
}
@@ -115,5 +117,5 @@
t.getTime( "Got aggregated values" );
return map;
- }
+ }
}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java 2012-12-27 19:15:15 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java 2012-12-30 16:23:41 +0000
@@ -29,6 +29,7 @@
import static org.hisp.dhis.dataelement.DataElement.AGGREGATION_OPERATOR_AVERAGE;
import static org.hisp.dhis.dataelement.DataElement.AGGREGATION_OPERATOR_SUM;
+import static org.hisp.dhis.analytics.AggregationType.*;
import java.util.ArrayList;
import java.util.Collection;
@@ -93,13 +94,28 @@
for ( DataQueryParams byAggregationType : groupedByAggregationType )
{
- byAggregationType.setTableName( byPartition.getTableName() );
- byAggregationType.setOrganisationUnitLevel( byOrgUnitLevel.getOrganisationUnitLevel() );
- byAggregationType.setPeriodType( byPeriodType.getPeriodType() );
-
- //TODO split on data element period type for average disaggregation
-
- queries.add( byAggregationType );
+ if ( AVERAGE_DISAGGREGATION.equals( byAggregationType.getAggregationType() ) )
+ {
+ List<DataQueryParams> groupedByDataPeriodType = groupByDataPeriodType( byAggregationType );
+
+ for ( DataQueryParams byDataPeriodType : groupedByDataPeriodType )
+ {
+ byDataPeriodType.setTableName( byPartition.getTableName() );
+ byDataPeriodType.setOrganisationUnitLevel( byOrgUnitLevel.getOrganisationUnitLevel() );
+ byDataPeriodType.setPeriodType( byPeriodType.getPeriodType() );
+ byDataPeriodType.setAggregationType( byAggregationType.getAggregationType() );
+
+ queries.add( byDataPeriodType );
+ }
+ }
+ else
+ {
+ byAggregationType.setTableName( byPartition.getTableName() );
+ byAggregationType.setOrganisationUnitLevel( byOrgUnitLevel.getOrganisationUnitLevel() );
+ byAggregationType.setPeriodType( byPeriodType.getPeriodType() );
+
+ queries.add( byAggregationType );
+ }
}
}
}
@@ -267,6 +283,17 @@
return queries;
}
+ /**
+ * Groups the given query in sub queries based on the aggregation type of its
+ * data elements. The aggregation type can be sum, average aggregation or
+ * average disaggregation. Sum means that the data elements have sum aggregation
+ * operator. Average aggregation means that the data elements have the average
+ * aggregation operator and that the period type of the data elements have
+ * higher or equal frequency than the aggregation period type. Average disaggregation
+ * means that the data elements have the average aggregation operator and
+ * that the period type of the data elements have lower frequency than the
+ * aggregation period type.
+ */
private List<DataQueryParams> groupByAggregationType( DataQueryParams params )
{
List<DataQueryParams> queries = new ArrayList<DataQueryParams>();
@@ -293,6 +320,33 @@
}
/**
+ * Groups the given query in sub queries based on the period type of its
+ * data elements. Sets the data period type on each query.
+ */
+ private List<DataQueryParams> groupByDataPeriodType( DataQueryParams params )
+ {
+ List<DataQueryParams> queries = new ArrayList<DataQueryParams>();
+
+ if ( params.getDatElements() == null || params.getDatElements().isEmpty() )
+ {
+ queries.add( new DataQueryParams( params ) );
+ return queries;
+ }
+
+ ListMap<PeriodType, String> periodTypeDataElementMap = getPeriodTypeDataElementMap( params.getDatElements() );
+
+ for ( PeriodType periodType : periodTypeDataElementMap.keySet() )
+ {
+ DataQueryParams query = new DataQueryParams( params );
+ query.setDataElements( periodTypeDataElementMap.get( periodType ) );
+ query.setDataPeriodType( periodType );
+ queries.add( query );
+ }
+
+ return queries;
+ }
+
+ /**
* Replaces the period filter with individual filters for each period type.
*/
private List<DataQueryParams> setFilterByPeriodType( List<DataQueryParams> queries )
@@ -379,7 +433,7 @@
return map;
}
-
+
/**
* Creates a mapping between the aggregation type and data element for the
* given data elements and period type.
@@ -413,4 +467,22 @@
return map;
}
+
+ /**
+ * Creates a mapping between the period type and the data element for the
+ * given data elements.
+ */
+ private ListMap<PeriodType, String> getPeriodTypeDataElementMap( Collection<String> dataElements )
+ {
+ ListMap<PeriodType, String> map = new ListMap<PeriodType, String>();
+
+ for ( String element : dataElements )
+ {
+ DataElement dataElement = dataElementService.getDataElement( element );
+
+ map.putValue( dataElement.getPeriodType(), element );
+ }
+
+ return map;
+ }
}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java 2012-12-27 19:15:15 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java 2012-12-30 16:23:41 +0000
@@ -27,14 +27,17 @@
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
+import static org.hisp.dhis.analytics.AggregationType.AVERAGE_AGGREGATION;
+import static org.hisp.dhis.analytics.AggregationType.AVERAGE_DISAGGREGATION;
import static org.hisp.dhis.analytics.DataQueryParams.VALUE_ID;
import static org.hisp.dhis.system.util.TextUtils.getCommaDelimitedString;
import static org.hisp.dhis.system.util.TextUtils.getQuotedCommaDelimitedString;
-import static org.hisp.dhis.analytics.AggregationType.*;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.Future;
import org.apache.commons.logging.Log;
@@ -42,12 +45,15 @@
import org.hisp.dhis.analytics.AnalyticsManager;
import org.hisp.dhis.analytics.DataQueryParams;
import org.hisp.dhis.period.PeriodType;
+import org.hisp.dhis.system.util.ListMap;
import org.hisp.dhis.system.util.SqlHelper;
+import org.hisp.dhis.system.util.TextUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.support.rowset.SqlRowSet;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.AsyncResult;
+import org.springframework.util.Assert;
/**
* This class is responsible for producing aggregated data values. It reads data
@@ -59,6 +65,8 @@
public class JdbcAnalyticsManager
implements AnalyticsManager
{
+ //TODO optimize when all options in dimensions are selected
+
private static final Log log = LogFactory.getLog( JdbcAnalyticsManager.class );
@Autowired
@@ -68,11 +76,12 @@
// Implementation
// -------------------------------------------------------------------------
- //TODO optimize when all options in dimensions are selected
-
@Async
public Future<Map<String, Double>> getAggregatedDataValues( DataQueryParams params )
{
+ ListMap<String, String> dataPeriodAggregationPeriodMap = params.getDataPeriodAggregationPeriodMap();
+ params.replaceAggregationPeriodsWithDataPeriods( dataPeriodAggregationPeriodMap );
+
List<String> dimensions = params.getDimensionNames();
Map<String, List<String>> dimensionMap = params.getDimensionMap();
@@ -85,7 +94,7 @@
sql += params.isAggregationType( AVERAGE_AGGREGATION ) ? "sum(daysxvalue) / " + days : "sum(value)";
sql += " as value from " + params.getTableName() + " ";
-
+
for ( String dim : dimensions )
{
sql += sqlHelper.whereAnd() + " " + dim + " in (" + getQuotedCommaDelimitedString( dimensionMap.get( dim ) ) + " ) ";
@@ -113,13 +122,47 @@
key.append( rowSet.getString( dim ) + SEP );
}
- key.deleteCharAt( key.length() - SEP.length() );
+ key.deleteCharAt( key.length() - 1 );
Double value = rowSet.getDouble( VALUE_ID );
map.put( key.toString(), value );
}
+ replaceDataPeriodsWithAggregationPeriods( map, params, dataPeriodAggregationPeriodMap );
+
return new AsyncResult<Map<String, Double>>( map );
- }
+ }
+
+ public void replaceDataPeriodsWithAggregationPeriods( Map<String, Double> dataValueMap, DataQueryParams params, ListMap<String, String> dataPeriodAggregationPeriodMap )
+ {
+ if ( params.isAggregationType( AVERAGE_DISAGGREGATION ) )
+ {
+ int periodIndex = params.getPeriodDimensionIndex();
+
+ Set<String> keys = new HashSet<String>( dataValueMap.keySet() );
+
+ for ( String key : keys )
+ {
+ String[] keyArray = key.split( String.valueOf( SEP ) );
+
+ Assert.notNull( keyArray[periodIndex], keyArray.toString() );
+
+ List<String> periods = dataPeriodAggregationPeriodMap.get( keyArray[periodIndex] );
+
+ Assert.notNull( periods, dataPeriodAggregationPeriodMap.toString() );
+
+ Double value = dataValueMap.get( key );
+
+ for ( String period : periods )
+ {
+ String[] keyCopy = keyArray.clone();
+ keyCopy[periodIndex] = period;
+ dataValueMap.put( TextUtils.toString( keyCopy, SEP ), value );
+ }
+
+ dataValueMap.remove( key );
+ }
+ }
+ }
}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java 2012-12-27 18:15:23 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java 2012-12-30 16:23:41 +0000
@@ -44,6 +44,9 @@
import org.hisp.dhis.organisationunit.OrganisationUnitService;
import org.hisp.dhis.period.Cal;
import org.hisp.dhis.period.PeriodType;
+import org.hisp.dhis.period.QuarterlyPeriodType;
+import org.hisp.dhis.period.YearlyPeriodType;
+import org.hisp.dhis.system.util.ListMap;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
@@ -104,6 +107,34 @@
// Tests
// -------------------------------------------------------------------------
+ public void testGetDataPeriodAggregationPeriodMap()
+ {
+ DataQueryParams params = new DataQueryParams();
+ params.setDataElements( Arrays.asList( deA.getUid(), deB.getUid(), deC.getUid(), deD.getUid() ) );
+ params.setOrganisationUnits( Arrays.asList( ouA.getUid(), ouB.getUid(), ouC.getUid(), ouD.getUid(), ouE.getUid() ) );
+ params.setPeriods( Arrays.asList( "2000Q1", "2000Q2", "2000Q3", "2000Q4", "2001Q1", "2001Q2" ) );
+ params.setPeriodType( QuarterlyPeriodType.NAME );
+ params.setDataPeriodType( new YearlyPeriodType() );
+
+ ListMap<String, String> map = params.getDataPeriodAggregationPeriodMap();
+
+ assertEquals( 2, map.size() );
+
+ assertTrue( map.keySet().contains( "2000" ) );
+ assertTrue( map.keySet().contains( "2001" ) );
+
+ assertEquals( 4, map.get( "2000" ).size() );
+ assertEquals( 2, map.get( "2001" ).size() );
+
+ assertTrue( map.get( "2000" ).contains( "2000Q1" ) );
+ assertTrue( map.get( "2000" ).contains( "2000Q2" ) );
+ assertTrue( map.get( "2000" ).contains( "2000Q3" ) );
+ assertTrue( map.get( "2000" ).contains( "2000Q4" ) );
+
+ assertTrue( map.get( "2001" ).contains( "2001Q1" ) );
+ assertTrue( map.get( "2001" ).contains( "2001Q2" ) );
+ }
+
/**
* Query spans 2 partitions. Splits in 2 queries for each partition, then
* splits in 2 queries on organisation units to satisfy optimal for a total
=== modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/TextUtils.java'
--- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/TextUtils.java 2012-12-18 16:01:44 +0000
+++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/TextUtils.java 2012-12-30 16:23:41 +0000
@@ -306,4 +306,29 @@
{
return string != null ? string.toLowerCase() : null;
}
+
+ /**
+ * Null-safe method for writing the items of a string array out as a string
+ * separated by the given char separator.
+ *
+ * @param array the array.
+ * @param separator the separator of the array items.
+ * @return a string.
+ */
+ public static String toString( String[] array, char separator )
+ {
+ StringBuilder builder = new StringBuilder();
+
+ if ( array != null && array.length > 0 )
+ {
+ for ( String string : array )
+ {
+ builder.append( string ).append( separator );
+ }
+
+ builder.deleteCharAt( builder.length() - 1 );
+ }
+
+ return builder.toString();
+ }
}
