dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #36910
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 18879: Analytics. Skipped exploding of indicator expressions and rather fetch data element values in pot...
Merge authors:
Lars Helge Øverland (larshelge)
------------------------------------------------------------
revno: 18879 [merge]
committer: Lars Helge Overland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Fri 2015-04-10 14:12:33 +0200
message:
Analytics. Skipped exploding of indicator expressions and rather fetch data element values in potentially two requests, one for totals, one for option combos. This allows for using data element totals in indicators which are captured using different category combos / disaggregations.
modified:
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java
dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java 2015-04-10 10:38:22 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java 2015-04-10 11:18:20 +0000
@@ -942,34 +942,37 @@
// -------------------------------------------------------------------------
/**
- * Returns a mapping of permutation keys and mappings of data element operands
- * and values, based on the given mapping of dimension option keys and
+ * Populates a mapping of permutation keys and mappings of data element operands
+ * and values based on the given mapping of dimension option keys and
* aggregated values. The data element dimension will be at index 0 and the
- * category option combo dimension will be at index 1.
+ * category option combo dimension will be at index 1, if category option
+ * combinations is enabled.
+ *
+ * @param permutationMap the map to populate with permutations.
+ * @param aggregatedDataMap the aggregated data map.
+ * @param cocEnabled indicates whether the given aggregated data map includes
+ * a category option combination dimension.
*/
- public static Map<String, Map<DataElementOperand, Double>> getPermutationOperandValueMap( Map<String, Double> aggregatedDataMap )
+ public static void putPermutationOperandValueMap( MapMap<String, DataElementOperand, Double> permutationMap,
+ Map<String, Double> aggregatedDataMap, boolean cocEnabled )
{
- MapMap<String, DataElementOperand, Double> valueMap = new MapMap<>();
-
for ( String key : aggregatedDataMap.keySet() )
{
List<String> keys = new ArrayList<>( Arrays.asList( key.split( DIMENSION_SEP ) ) );
String de = keys.get( DE_IN_INDEX );
- String coc = keys.get( CO_IN_INDEX );
+ String coc = cocEnabled ? keys.get( CO_IN_INDEX ) : null;
DataElementOperand operand = new DataElementOperand( de, coc );
- ListUtils.removeAll( keys, DE_IN_INDEX, CO_IN_INDEX );
+ ListUtils.removeAll( keys, DE_IN_INDEX, ( cocEnabled ? CO_IN_INDEX : -1 ) );
String permKey = StringUtils.join( keys, DIMENSION_SEP );
Double value = aggregatedDataMap.get( key );
- valueMap.putEntry( permKey, operand, value );
+ permutationMap.putEntry( permKey, operand, value );
}
-
- return valueMap;
}
/**
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java 2015-04-10 10:38:22 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java 2015-04-10 11:54:45 +0000
@@ -106,6 +106,7 @@
import org.hisp.dhis.common.IdentifiableObjectUtils;
import org.hisp.dhis.common.IdentifiableProperty;
import org.hisp.dhis.common.IllegalQueryException;
+import org.hisp.dhis.common.MapMap;
import org.hisp.dhis.common.NameableObject;
import org.hisp.dhis.common.NameableObjectUtils;
import org.hisp.dhis.constant.ConstantService;
@@ -286,8 +287,6 @@
int indicatorIndex = params.getIndicatorDimensionIndex();
List<Indicator> indicators = asTypedList( params.getIndicators() );
- expressionService.explodeExpressions( indicators );
-
// -----------------------------------------------------------------
// Get indicator values
// -----------------------------------------------------------------
@@ -1225,34 +1224,68 @@
*/
private Map<String, Map<DataElementOperand, Double>> getPermutationOperandValueMap( DataQueryParams params )
{
- DataQueryParams dataSourceParams = getQueryIndicatorsReplacedByDataElements( params );
-
- Map<String, Double> aggregatedDataMap = getAggregatedDataValueMap( dataSourceParams );
-
- return DataQueryParams.getPermutationOperandValueMap( aggregatedDataMap );
- }
-
- /**
- * Returns a new instance of the given query where indicators are replaced
- * with the data elements part of the indicator expressions.
- *
- * @param params the data query parameters.
- * @param indicatorIndex the index of the indicator dimension in the given query.
- * @return the data query parameters.
- */
- private DataQueryParams getQueryIndicatorsReplacedByDataElements( DataQueryParams params )
- {
- List<Indicator> indicators = asTypedList( params.getIndicators() );
- List<NameableObject> dataElements = asList( expressionService.getDataElementsInIndicators( indicators ) );
-
- DataQueryParams dataSourceParams = params.instance().removeDimensions( DATAELEMENT_DIM_ID, DATASET_DIM_ID, INDICATOR_DIM_ID );
-
- dataSourceParams.getDimensions().add( DataQueryParams.DE_IN_INDEX, new BaseDimensionalObject(
- DATAELEMENT_DIM_ID, DimensionType.DATAELEMENT, dataElements ) );
- dataSourceParams.getDimensions().add( DataQueryParams.CO_IN_INDEX, new BaseDimensionalObject(
- CATEGORYOPTIONCOMBO_DIM_ID, DimensionType.CATEGORY_OPTION_COMBO, new ArrayList<NameableObject>() ) );
-
- return dataSourceParams;
+ Map<String, Double> aggregatedDataTotalsMap = getAggregatedDataValueMapTotals( params );
+ Map<String, Double> aggregatedDataOptionCombosMap = getAggregatedDataValueMapOptionCombos( params );
+
+ MapMap<String, DataElementOperand, Double> permOperandValueMap = new MapMap<>();
+
+ DataQueryParams.putPermutationOperandValueMap( permOperandValueMap, aggregatedDataTotalsMap, false );
+ DataQueryParams.putPermutationOperandValueMap( permOperandValueMap, aggregatedDataOptionCombosMap, true );
+
+ return permOperandValueMap;
+ }
+
+ /**
+ * Returns a mapping of dimension keys and aggregated values for the data
+ * element totals part of the indicators in the given query.
+ *
+ * @param params the data query parameters.
+ * @return a mapping of dimension keys and aggregated values.
+ */
+ private Map<String, Double> getAggregatedDataValueMapTotals( DataQueryParams params )
+ {
+ List<Indicator> indicators = asTypedList( params.getIndicators() );
+ List<NameableObject> dataElements = asList( expressionService.getDataElementTotalsInIndicators( indicators ) );
+
+ if ( !dataElements.isEmpty() )
+ {
+ DataQueryParams dataSourceParams = params.instance().removeDimensions( DATAELEMENT_DIM_ID, DATASET_DIM_ID, INDICATOR_DIM_ID );
+
+ dataSourceParams.getDimensions().add( DataQueryParams.DE_IN_INDEX, new BaseDimensionalObject(
+ DATAELEMENT_DIM_ID, DimensionType.DATAELEMENT, dataElements ) );
+
+ return getAggregatedDataValueMap( dataSourceParams );
+ }
+
+ return new HashMap<>();
+ }
+
+ /**
+ * Returns a mapping of dimension keys and aggregated values for the data
+ * elements with category option combinations part of the indicators in the
+ * given query.
+ *
+ * @param params the data query parameters.
+ * @return a mapping of dimension keys and aggregated values.
+ */
+ private Map<String, Double> getAggregatedDataValueMapOptionCombos( DataQueryParams params )
+ {
+ List<Indicator> indicators = asTypedList( params.getIndicators() );
+ List<NameableObject> dataElements = asList( expressionService.getDataElementWithOptionCombosInIndicators( indicators ) );
+
+ if ( !dataElements.isEmpty() )
+ {
+ DataQueryParams dataSourceParams = params.instance().removeDimensions( DATAELEMENT_DIM_ID, DATASET_DIM_ID, INDICATOR_DIM_ID );
+
+ dataSourceParams.getDimensions().add( DataQueryParams.DE_IN_INDEX, new BaseDimensionalObject(
+ DATAELEMENT_DIM_ID, DimensionType.DATAELEMENT, dataElements ) );
+ dataSourceParams.getDimensions().add( DataQueryParams.CO_IN_INDEX, new BaseDimensionalObject(
+ CATEGORYOPTIONCOMBO_DIM_ID, DimensionType.CATEGORY_OPTION_COMBO, new ArrayList<NameableObject>() ) );
+
+ return getAggregatedDataValueMap( dataSourceParams );
+ }
+
+ return new HashMap<>();
}
/**
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java 2015-04-10 10:38:22 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java 2015-04-10 11:18:20 +0000
@@ -58,6 +58,7 @@
import org.hisp.dhis.common.DimensionalObject;
import org.hisp.dhis.common.IllegalQueryException;
import org.hisp.dhis.common.ListMap;
+import org.hisp.dhis.common.MapMap;
import org.hisp.dhis.common.NameableObject;
import org.hisp.dhis.dataelement.DataElement;
import org.hisp.dhis.dataelement.DataElementCategoryOptionCombo;
@@ -249,19 +250,9 @@
assertEquals( pesB, paramsB.getPeriods() );
}
- /**
- * Data element dimension must be at index 0 and category option combo
- * dimension must be at index 1 in map.
- */
@Test
- public void testGetPermutationOperandValueMap()
+ public void testGetPermutationOperandValueMapCocEnabled()
{
- DataQueryParams params = new DataQueryParams();
- params.setDataElements( getList( deA, deB ) );
- params.setOrganisationUnits( getList( ouA, ouB ) );
- params.setPeriods( getList( createPeriod( "2000Q1" ), createPeriod( "2000Q2" ) ) );
- params.enableCategoryOptionCombos();
-
Map<String, Double> aggregatedDataMap = new HashMap<>();
aggregatedDataMap.put( deA.getUid() + DIMENSION_SEP + coc.getUid() + DIMENSION_SEP + ouA.getUid() + DIMENSION_SEP + "2000Q1", 1d );
aggregatedDataMap.put( deA.getUid() + DIMENSION_SEP + coc.getUid() + DIMENSION_SEP + ouA.getUid() + DIMENSION_SEP + "2000Q2", 2d );
@@ -272,7 +263,9 @@
aggregatedDataMap.put( deB.getUid() + DIMENSION_SEP + coc.getUid() + DIMENSION_SEP + ouB.getUid() + DIMENSION_SEP + "2000Q1", 7d );
aggregatedDataMap.put( deB.getUid() + DIMENSION_SEP + coc.getUid() + DIMENSION_SEP + ouB.getUid() + DIMENSION_SEP + "2000Q2", 8d );
- Map<String, Map<DataElementOperand, Double>> permutationMap = DataQueryParams.getPermutationOperandValueMap( aggregatedDataMap );
+ MapMap<String, DataElementOperand, Double> permutationMap = new MapMap<>();
+
+ DataQueryParams.putPermutationOperandValueMap( permutationMap, aggregatedDataMap, true );
assertNotNull( permutationMap );
@@ -315,7 +308,128 @@
assertEquals( ouBQ1Expected, ouBQ1 );
assertEquals( ouBQ2Expected, ouBQ2 );
}
-
+
+ @Test
+ public void testGetPermutationOperandValueMapCocDisabled()
+ {
+ Map<String, Double> aggregatedDataMap = new HashMap<>();
+ aggregatedDataMap.put( deA.getUid() + DIMENSION_SEP + ouA.getUid() + DIMENSION_SEP + "200101", 1d );
+ aggregatedDataMap.put( deA.getUid() + DIMENSION_SEP + ouA.getUid() + DIMENSION_SEP + "200102", 2d );
+ aggregatedDataMap.put( deA.getUid() + DIMENSION_SEP + ouB.getUid() + DIMENSION_SEP + "200101", 3d );
+ aggregatedDataMap.put( deA.getUid() + DIMENSION_SEP + ouB.getUid() + DIMENSION_SEP + "200102", 4d );
+ aggregatedDataMap.put( deB.getUid() + DIMENSION_SEP + ouA.getUid() + DIMENSION_SEP + "200101", 5d );
+ aggregatedDataMap.put( deB.getUid() + DIMENSION_SEP + ouA.getUid() + DIMENSION_SEP + "200102", 6d );
+ aggregatedDataMap.put( deB.getUid() + DIMENSION_SEP + ouB.getUid() + DIMENSION_SEP + "200101", 7d );
+ aggregatedDataMap.put( deB.getUid() + DIMENSION_SEP + ouB.getUid() + DIMENSION_SEP + "200102", 8d );
+
+ MapMap<String, DataElementOperand, Double> permutationMap = new MapMap<>();
+
+ DataQueryParams.putPermutationOperandValueMap( permutationMap, aggregatedDataMap, false );
+
+ assertNotNull( permutationMap );
+
+ String ouAM1Key = ouA.getUid() + DIMENSION_SEP + "200101";
+ String ouAM2Key = ouA.getUid() + DIMENSION_SEP + "200102";
+ String ouBM1Key = ouB.getUid() + DIMENSION_SEP + "200101";
+ String ouBM2Key = ouB.getUid() + DIMENSION_SEP + "200102";
+
+ Map<DataElementOperand, Double> ouAM1 = permutationMap.get( ouAM1Key );
+ Map<DataElementOperand, Double> ouAM2 = permutationMap.get( ouAM2Key );
+ Map<DataElementOperand, Double> ouBM1 = permutationMap.get( ouBM1Key );
+ Map<DataElementOperand, Double> ouBM2 = permutationMap.get( ouBM2Key );
+
+ assertEquals( 2, ouAM1.size() );
+ assertEquals( 2, ouAM2.size() );
+ assertEquals( 2, ouBM1.size() );
+ assertEquals( 2, ouBM2.size() );
+
+ DataElementOperand deACoc = new DataElementOperand( deA.getUid(), null );
+ DataElementOperand deBCoc = new DataElementOperand( deB.getUid(), null );
+
+ Map<DataElementOperand, Double> ouAM1Expected = new HashMap<>();
+ ouAM1Expected.put( deACoc, 1d );
+ ouAM1Expected.put( deBCoc, 5d );
+
+ Map<DataElementOperand, Double> ouAM2Expected = new HashMap<>();
+ ouAM2Expected.put( deACoc, 2d );
+ ouAM2Expected.put( deBCoc, 6d );
+
+ Map<DataElementOperand, Double> ouBM1Expected = new HashMap<>();
+ ouBM1Expected.put( deACoc, 3d );
+ ouBM1Expected.put( deBCoc, 7d );
+
+ Map<DataElementOperand, Double> ouBM2Expected = new HashMap<>();
+ ouBM2Expected.put( deACoc, 4d );
+ ouBM2Expected.put( deBCoc, 8d );
+
+ assertEquals( ouAM1Expected, ouAM1 );
+ assertEquals( ouAM2Expected, ouAM2 );
+ assertEquals( ouBM1Expected, ouBM1 );
+ assertEquals( ouBM2Expected, ouBM2 );
+ }
+
+ @Test
+ public void testGetPermutationOperandValueMap()
+ {
+ Map<String, Double> aggregatedTotalsDataMap = new HashMap<>();
+ aggregatedTotalsDataMap.put( deA.getUid() + DIMENSION_SEP + ouA.getUid() + DIMENSION_SEP + "2000Q1", 1d );
+ aggregatedTotalsDataMap.put( deA.getUid() + DIMENSION_SEP + ouA.getUid() + DIMENSION_SEP + "2000Q2", 2d );
+ aggregatedTotalsDataMap.put( deA.getUid() + DIMENSION_SEP + ouB.getUid() + DIMENSION_SEP + "2000Q1", 3d );
+ aggregatedTotalsDataMap.put( deA.getUid() + DIMENSION_SEP + ouB.getUid() + DIMENSION_SEP + "2000Q2", 4d );
+
+ Map<String, Double> aggregatedCocDataMap = new HashMap<>();
+ aggregatedCocDataMap.put( deB.getUid() + DIMENSION_SEP + coc.getUid() + DIMENSION_SEP + ouA.getUid() + DIMENSION_SEP + "2000Q1", 5d );
+ aggregatedCocDataMap.put( deB.getUid() + DIMENSION_SEP + coc.getUid() + DIMENSION_SEP + ouA.getUid() + DIMENSION_SEP + "2000Q2", 6d );
+ aggregatedCocDataMap.put( deB.getUid() + DIMENSION_SEP + coc.getUid() + DIMENSION_SEP + ouB.getUid() + DIMENSION_SEP + "2000Q1", 7d );
+ aggregatedCocDataMap.put( deB.getUid() + DIMENSION_SEP + coc.getUid() + DIMENSION_SEP + ouB.getUid() + DIMENSION_SEP + "2000Q2", 8d );
+
+ MapMap<String, DataElementOperand, Double> permutationMap = new MapMap<>();
+
+ DataQueryParams.putPermutationOperandValueMap( permutationMap, aggregatedTotalsDataMap, false );
+ DataQueryParams.putPermutationOperandValueMap( permutationMap, aggregatedCocDataMap, true );
+
+ assertNotNull( permutationMap );
+
+ String ouAQ1Key = ouA.getUid() + DIMENSION_SEP + "2000Q1";
+ String ouAQ2Key = ouA.getUid() + DIMENSION_SEP + "2000Q2";
+ String ouBQ1Key = ouB.getUid() + DIMENSION_SEP + "2000Q1";
+ String ouBQ2Key = ouB.getUid() + DIMENSION_SEP + "2000Q2";
+
+ Map<DataElementOperand, Double> ouAQ1 = permutationMap.get( ouAQ1Key );
+ Map<DataElementOperand, Double> ouAQ2 = permutationMap.get( ouAQ2Key );
+ Map<DataElementOperand, Double> ouBQ1 = permutationMap.get( ouBQ1Key );
+ Map<DataElementOperand, Double> ouBQ2 = permutationMap.get( ouBQ2Key );
+
+ assertEquals( 2, ouAQ1.size() );
+ assertEquals( 2, ouAQ2.size() );
+ assertEquals( 2, ouBQ1.size() );
+ assertEquals( 2, ouBQ2.size() );
+
+ DataElementOperand deACoc = new DataElementOperand( deA.getUid(), null );
+ DataElementOperand deBCoc = new DataElementOperand( deB.getUid(), coc.getUid() );
+
+ Map<DataElementOperand, Double> ouAQ1Expected = new HashMap<>();
+ ouAQ1Expected.put( deACoc, 1d );
+ ouAQ1Expected.put( deBCoc, 5d );
+
+ Map<DataElementOperand, Double> ouAQ2Expected = new HashMap<>();
+ ouAQ2Expected.put( deACoc, 2d );
+ ouAQ2Expected.put( deBCoc, 6d );
+
+ Map<DataElementOperand, Double> ouBQ1Expected = new HashMap<>();
+ ouBQ1Expected.put( deACoc, 3d );
+ ouBQ1Expected.put( deBCoc, 7d );
+
+ Map<DataElementOperand, Double> ouBQ2Expected = new HashMap<>();
+ ouBQ2Expected.put( deACoc, 4d );
+ ouBQ2Expected.put( deBCoc, 8d );
+
+ assertEquals( ouAQ1Expected, ouAQ1 );
+ assertEquals( ouAQ2Expected, ouAQ2 );
+ assertEquals( ouBQ1Expected, ouBQ1 );
+ assertEquals( ouBQ2Expected, ouBQ2 );
+ }
+
/**
* Ignores data element dimension and generates 2 x 3 = 6 combinations based
* on organisation unit and period dimensions.
Follow ups
