dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #22711
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 11003: Analytics, improved performance by running queries with different aggregation type in sequence. T...
------------------------------------------------------------
revno: 11003
committer: Lars Helge Øverland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Sun 2013-05-26 16:06:51 +0200
message:
Analytics, improved performance by running queries with different aggregation type in sequence. Typically with indicators there are many sum and few average data elements/queries, so being able to run the sum queries with optimal no of parallel queries improves response time.
added:
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryGroups.java
dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/DataQueryGroupsTest.java
modified:
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/QueryPlanner.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java
dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java
dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/Timer.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryGroups.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryGroups.java 1970-01-01 00:00:00 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryGroups.java 2013-05-26 14:06:51 +0000
@@ -0,0 +1,126 @@
+package org.hisp.dhis.analytics;
+
+/*
+ * Copyright (c) 2004-2012, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.hisp.dhis.common.ListMap;
+
+/**
+ * @author Lars Helge Overland
+ */
+public class DataQueryGroups
+{
+ private List<DataQueryParams> queries = new ArrayList<DataQueryParams>();
+
+ private List<List<DataQueryParams>> sequentialQueries = new ArrayList<List<DataQueryParams>>();
+
+ // -------------------------------------------------------------------------
+ // Constructor
+ // -------------------------------------------------------------------------
+
+ public DataQueryGroups( List<DataQueryParams> queries )
+ {
+ this.queries = queries;
+ this.sequentialQueries.addAll( getListMap( queries ).values() );
+ }
+
+ // -------------------------------------------------------------------------
+ // Public methods
+ // -------------------------------------------------------------------------
+
+ /**
+ * Gets all queries for all groups.
+ */
+ public List<DataQueryParams> getAllQueries()
+ {
+ return queries;
+ }
+
+ /**
+ * Gets groups of queries which should be run in sequence for optimal
+ * performance. Currently queries with different aggregation type are run
+ * in sequence due to the typical indicator query, where few data elements
+ * have the average aggregation operator and many have the sum. Performance
+ * will increase if optimal number of queries can be run in parallel for the
+ * queries which take most time, which is in this case are the ones with
+ * sum aggregation type.
+ */
+ public List<List<DataQueryParams>> getSequentialQueries()
+ {
+ return sequentialQueries;
+ }
+
+ /**
+ * Indicates whether the current state of the query groups is optimal. Uses
+ * the given optimal query number compared to the size of the largest query
+ * group to determine the outcome.
+ */
+ public boolean isOptimal( int optimalQueries )
+ {
+ return getLargestGroupSize() >= optimalQueries;
+ }
+
+ /**
+ * Gets the size of the larges query group.
+ */
+ public int getLargestGroupSize()
+ {
+ int max = 0;
+
+ for ( List<DataQueryParams> list : sequentialQueries )
+ {
+ max = list.size() > max ? list.size() : max;
+ }
+
+ return max;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "[Seq queries: " + sequentialQueries.size() + ", all queries: " + queries.size() + "]";
+ }
+
+ // -------------------------------------------------------------------------
+ // Supportive methods
+ // -------------------------------------------------------------------------
+
+ private static ListMap<String, DataQueryParams> getListMap( List<DataQueryParams> queries )
+ {
+ ListMap<String, DataQueryParams> map = new ListMap<String, DataQueryParams>();
+
+ for ( DataQueryParams query : queries )
+ {
+ map.putValue( query.getSequentialQueryGroupKey(), query );
+ }
+
+ return map;
+ }
+}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java 2013-05-25 13:59:49 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java 2013-05-26 14:06:51 +0000
@@ -166,6 +166,17 @@
}
/**
+ * Returns a key representing a group of queries which should be run in
+ * sequence. Currently queries with different aggregation type are run in
+ * sequence. It is not allowed for the implementation to differentiate on
+ * dimensional objects. TODO test including tableName (partition)
+ */
+ public String getSequentialQueryGroupKey()
+ {
+ return aggregationType != null ? aggregationType.toString() : null;
+ }
+
+ /**
* Indicates whether the filters of this query spans more than one partition.
* If true it means that a period filter exists and that the periods span
* multiple years.
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/QueryPlanner.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/QueryPlanner.java 2013-05-23 16:57:03 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/QueryPlanner.java 2013-05-26 14:06:51 +0000
@@ -59,22 +59,27 @@
throws IllegalQueryException;
/**
- * Creates a list of DataQueryParams. It is mandatory to group the queries by
- * the following criteria: 1) partition / year 2) period type 3) organisation
- * unit level. If the number of queries produced by this grouping is equal or
- * larger than the number of optimal queries, those queries are returned. Next
- * splits on organisation unit dimension, and returns if optimal queries are
- * satisfied. Next splits on data element dimension, and returns if optimal
- * queries are satisfied.
- *
- * Does not attempt to split on period or organisation unit group set dimensions,
- * as splitting on columns with low cardinality typically decreases performance.
- *
- * @param params the data query params.
- * @param optimalQueries the number of optimal queries for the planner to return.
+ * Creates a DataQueryGroups object. It is mandatory to group the queries by
+ * the following criteria: 1) partition / year 2) organisation unit level
+ * 3) period type 4) aggregation type. The DataQueryGroups contains groups of
+ * queries. The query groups should be run in sequence while the queries within
+ * each group should be run in parallel for optimal performance.
+ *
+ * If the number of queries produced by this grouping is equal or
+ * larger than the number of optimal queries, those queries are returned. If
+ * not it will split on the data element dimension, data set dimension and
+ * organisation unit dimension, and return immediately after each step if
+ * optimal queries are met.
+ *
+ * It does not attempt to split on period dimension as splitting on columns
+ * with low cardinality typically does not improve performance.
+ *
+ * @param params the data query parameters.
+ * @param optimalQueries the number of optimal queries for the planner to
+ * return for each query group.
* @param tableName the base table name.
- * @return list of data query params.
+ * @return a DataQueryGroups object.
*/
- List<DataQueryParams> planQuery( DataQueryParams params, int optimalQueries, String tableName )
+ DataQueryGroups planQuery( DataQueryParams params, int optimalQueries, String tableName )
throws IllegalQueryException;
}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java 2013-05-25 12:22:29 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java 2013-05-26 14:06:51 +0000
@@ -73,6 +73,7 @@
import org.hisp.dhis.analytics.AggregationType;
import org.hisp.dhis.analytics.AnalyticsManager;
import org.hisp.dhis.analytics.AnalyticsService;
+import org.hisp.dhis.analytics.DataQueryGroups;
import org.hisp.dhis.analytics.DataQueryParams;
import org.hisp.dhis.analytics.DimensionItem;
import org.hisp.dhis.analytics.IllegalQueryException;
@@ -495,40 +496,45 @@
Timer t = new Timer().start();
- List<DataQueryParams> queries = queryPlanner.planQuery( params, optimalQueries, tableName );
-
- t.getSplitTime( "Planned query, got: " + queries.size() + " for optimal: " + optimalQueries );
-
- List<Future<Map<String, Double>>> futures = new ArrayList<Future<Map<String, Double>>>();
-
- for ( DataQueryParams query : queries )
- {
- futures.add( analyticsManager.getAggregatedDataValues( query ) );
- }
+ DataQueryGroups queryGroups = queryPlanner.planQuery( params, optimalQueries, tableName );
+
+ t.getSplitTime( "Planned query, got: " + queryGroups.getLargestGroupSize() + " for optimal: " + optimalQueries );
Map<String, Double> map = new HashMap<String, Double>();
- for ( Future<Map<String, Double>> future : futures )
+ for ( List<DataQueryParams> queries : queryGroups.getSequentialQueries() )
{
- try
- {
- Map<String, Double> taskValues = future.get();
-
- if ( taskValues != null )
- {
- map.putAll( taskValues );
- }
- }
- catch ( Exception ex )
- {
- log.error( DebugUtils.getStackTrace( ex ) );
- log.error( DebugUtils.getStackTrace( ex.getCause() ) );
-
- throw new RuntimeException( "Error during execution of aggregation query task", ex );
- }
+ List<Future<Map<String, Double>>> futures = new ArrayList<Future<Map<String, Double>>>();
+
+ for ( DataQueryParams query : queries )
+ {
+ futures.add( analyticsManager.getAggregatedDataValues( query ) );
+ }
+
+ for ( Future<Map<String, Double>> future : futures )
+ {
+ try
+ {
+ Map<String, Double> taskValues = future.get();
+
+ if ( taskValues != null )
+ {
+ map.putAll( taskValues );
+ }
+ }
+ catch ( Exception ex )
+ {
+ log.error( DebugUtils.getStackTrace( ex ) );
+ log.error( DebugUtils.getStackTrace( ex.getCause() ) );
+
+ throw new RuntimeException( "Error during execution of aggregation query task", ex );
+ }
+ }
+
+ t.getSplitTime( "Got aggregated values for query group" );
}
- t.getTime( "Got aggregated value" );
+ t.getTime( "Got aggregated values" );
return map;
}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java 2013-05-25 13:59:49 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java 2013-05-26 14:06:51 +0000
@@ -45,6 +45,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.hisp.dhis.analytics.AggregationType;
+import org.hisp.dhis.analytics.DataQueryGroups;
import org.hisp.dhis.analytics.DataQueryParams;
import org.hisp.dhis.common.DimensionType;
import org.hisp.dhis.analytics.IllegalQueryException;
@@ -72,7 +73,6 @@
{
private static final Log log = LogFactory.getLog( DefaultQueryPlanner.class );
- //TODO call getLevelOrgUnitMap once?
//TODO shortcut group by methods when only 1 option?
@Autowired
@@ -179,7 +179,7 @@
}
}
- public List<DataQueryParams> planQuery( DataQueryParams params, int optimalQueries, String tableName )
+ public DataQueryGroups planQuery( DataQueryParams params, int optimalQueries, String tableName )
{
validate( params );
@@ -232,40 +232,42 @@
}
}
- if ( queries.size() >= optimalQueries )
+ DataQueryGroups queryGroups = new DataQueryGroups( queries );
+
+ if ( queryGroups.isOptimal( optimalQueries ) )
{
- return queries;
+ return queryGroups;
}
// ---------------------------------------------------------------------
// Group by data element
// ---------------------------------------------------------------------
- queries = splitByDimension( queries, DATAELEMENT_DIM_ID, optimalQueries );
+ queryGroups = splitByDimension( queryGroups, DATAELEMENT_DIM_ID, optimalQueries );
- if ( queries.size() >= optimalQueries )
+ if ( queryGroups.isOptimal( optimalQueries ) )
{
- return queries;
+ return queryGroups;
}
// ---------------------------------------------------------------------
// Group by data set
// ---------------------------------------------------------------------
- queries = splitByDimension( queries, DATASET_DIM_ID, optimalQueries );
+ queryGroups = splitByDimension( queryGroups, DATASET_DIM_ID, optimalQueries );
- if ( queries.size() >= optimalQueries )
+ if ( queryGroups.isOptimal( optimalQueries ) )
{
- return queries;
+ return queryGroups;
}
// ---------------------------------------------------------------------
// Group by organisation unit
// ---------------------------------------------------------------------
- queries = splitByDimension( queries, ORGUNIT_DIM_ID, optimalQueries );
+ queryGroups = splitByDimension( queryGroups, ORGUNIT_DIM_ID, optimalQueries );
- return queries;
+ return queryGroups;
}
public boolean canQueryFromDataMart( DataQueryParams params )
@@ -280,13 +282,13 @@
/**
* Splits the given list of queries in sub queries on the given dimension.
*/
- private List<DataQueryParams> splitByDimension( List<DataQueryParams> queries, String dimension, int optimalQueries )
+ private DataQueryGroups splitByDimension( DataQueryGroups queryGroups, String dimension, int optimalQueries )
{
- int optimalForSubQuery = MathUtils.divideToFloor( optimalQueries, queries.size() );
+ int optimalForSubQuery = MathUtils.divideToFloor( optimalQueries, queryGroups.getLargestGroupSize() );
List<DataQueryParams> subQueries = new ArrayList<DataQueryParams>();
- for ( DataQueryParams query : queries )
+ for ( DataQueryParams query : queryGroups.getAllQueries() )
{
DimensionalObject dim = query.getDimension( dimension );
@@ -308,12 +310,12 @@
}
}
- if ( subQueries.size() > queries.size() )
+ if ( subQueries.size() > queryGroups.getAllQueries().size() )
{
- log.info( "Split on " + dimension + ": " + ( subQueries.size() / queries.size() ) );
+ log.info( "Split on " + dimension + ": " + ( subQueries.size() / queryGroups.getAllQueries().size() ) );
}
- return subQueries;
+ return new DataQueryGroups( subQueries );
}
// -------------------------------------------------------------------------
=== added file 'dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/DataQueryGroupsTest.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/DataQueryGroupsTest.java 1970-01-01 00:00:00 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/DataQueryGroupsTest.java 2013-05-26 14:06:51 +0000
@@ -0,0 +1,135 @@
+package org.hisp.dhis.analytics;
+
+/*
+ * Copyright (c) 2004-2012, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+import static org.hisp.dhis.common.NameableObjectUtils.getList;
+import static org.hisp.dhis.dataelement.DataElement.AGGREGATION_OPERATOR_AVERAGE;
+import static org.hisp.dhis.dataelement.DataElement.AGGREGATION_OPERATOR_SUM;
+import static org.hisp.dhis.dataelement.DataElement.VALUE_TYPE_INT;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.hisp.dhis.DhisConvenienceTest;
+import org.hisp.dhis.dataelement.DataElement;
+import org.hisp.dhis.organisationunit.OrganisationUnit;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * @author Lars Helge Overland
+ */
+public class DataQueryGroupsTest
+ extends DhisConvenienceTest
+{
+ // -------------------------------------------------------------------------
+ // Fixture
+ // -------------------------------------------------------------------------
+
+ private DataElement deA;
+ private DataElement deB;
+ private DataElement deC;
+ private DataElement deD;
+ private DataElement deE;
+ private DataElement deF;
+ private DataElement deG;
+
+ private OrganisationUnit ouA;
+ private OrganisationUnit ouB;
+ private OrganisationUnit ouC;
+ private OrganisationUnit ouD;
+ private OrganisationUnit ouE;
+
+ // -------------------------------------------------------------------------
+ // Tests
+ // -------------------------------------------------------------------------
+
+ @Before
+ public void before()
+ {
+ deA = createDataElement( 'A', VALUE_TYPE_INT, AGGREGATION_OPERATOR_SUM );
+ deB = createDataElement( 'B', VALUE_TYPE_INT, AGGREGATION_OPERATOR_SUM );
+ deC = createDataElement( 'C', VALUE_TYPE_INT, AGGREGATION_OPERATOR_SUM );
+ deD = createDataElement( 'D', VALUE_TYPE_INT, AGGREGATION_OPERATOR_SUM );
+ deE = createDataElement( 'E', VALUE_TYPE_INT, AGGREGATION_OPERATOR_SUM );
+ deF = createDataElement( 'F', VALUE_TYPE_INT, AGGREGATION_OPERATOR_AVERAGE );
+ deG = createDataElement( 'G', VALUE_TYPE_INT, AGGREGATION_OPERATOR_AVERAGE );
+
+ ouA = createOrganisationUnit( 'A' );
+ ouB = createOrganisationUnit( 'B' );
+ ouC = createOrganisationUnit( 'C' );
+ ouD = createOrganisationUnit( 'D' );
+ ouE = createOrganisationUnit( 'E' );
+ }
+
+ @Test
+ public void planQueryA()
+ {
+ DataQueryParams paramsA = new DataQueryParams();
+ paramsA.setDataElements( getList( deA, deB ) );
+ paramsA.setOrganisationUnits( getList( ouA, ouB, ouC, ouD, ouE ) );
+ paramsA.setPeriods( getList( createPeriod( "2000Q1" ), createPeriod( "2000Q2" ), createPeriod( "2000Q3" ), createPeriod( "2000Q4" ), createPeriod( "2001Q1" ), createPeriod( "2001Q2" ) ) );
+ paramsA.setAggregationType( AggregationType.SUM );
+
+ DataQueryParams paramsB = new DataQueryParams();
+ paramsB.setDataElements( getList( deC, deD ) );
+ paramsB.setOrganisationUnits( getList( ouA, ouB, ouC, ouD, ouE ) );
+ paramsB.setPeriods( getList( createPeriod( "2000Q1" ), createPeriod( "2000Q2" ), createPeriod( "2000Q3" ), createPeriod( "2000Q4" ), createPeriod( "2001Q1" ), createPeriod( "2001Q2" ) ) );
+ paramsB.setAggregationType( AggregationType.SUM );
+
+ DataQueryParams paramsC = new DataQueryParams();
+ paramsC.setDataElements( getList( deE ) );
+ paramsC.setOrganisationUnits( getList( ouA, ouB, ouC, ouD, ouE ) );
+ paramsC.setPeriods( getList( createPeriod( "2000Q1" ), createPeriod( "2000Q2" ), createPeriod( "2000Q3" ), createPeriod( "2000Q4" ), createPeriod( "2001Q1" ), createPeriod( "2001Q2" ) ) );
+ paramsC.setAggregationType( AggregationType.SUM );
+
+ DataQueryParams paramsD = new DataQueryParams();
+ paramsD.setDataElements( getList( deF, deG ) );
+ paramsD.setOrganisationUnits( getList( ouA, ouB, ouC, ouD, ouE ) );
+ paramsD.setPeriods( getList( createPeriod( "2000Q1" ), createPeriod( "2000Q2" ), createPeriod( "2000Q3" ), createPeriod( "2000Q4" ), createPeriod( "2001Q1" ), createPeriod( "2001Q2" ) ) );
+ paramsD.setAggregationType( AggregationType.AVERAGE_INT );
+
+ List<DataQueryParams> queries = new ArrayList<DataQueryParams>();
+ queries.add( paramsA );
+ queries.add( paramsB );
+ queries.add( paramsC );
+ queries.add( paramsD );
+
+ DataQueryGroups queryGroups = new DataQueryGroups( queries );
+
+ assertEquals( 2, queryGroups.getSequentialQueries().size() );
+ assertEquals( 4, queryGroups.getAllQueries().size() );
+ assertEquals( 3, queryGroups.getLargestGroupSize() );
+ assertTrue( queryGroups.isOptimal( 3 ) );
+ assertTrue( queryGroups.isOptimal( 2 ) );
+ assertFalse( queryGroups.isOptimal( 4 ) );
+ }
+}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java 2013-05-23 16:57:03 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java 2013-05-26 14:06:51 +0000
@@ -33,6 +33,7 @@
import static org.hisp.dhis.common.DimensionalObject.ORGUNIT_DIM_ID;
import static org.hisp.dhis.common.DimensionalObject.PERIOD_DIM_ID;
import static org.hisp.dhis.common.NameableObjectUtils.getList;
+import static org.hisp.dhis.dataelement.DataElement.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -44,6 +45,7 @@
import java.util.Map;
import org.hisp.dhis.DhisSpringTest;
+import org.hisp.dhis.analytics.DataQueryGroups;
import org.hisp.dhis.analytics.DataQueryParams;
import org.hisp.dhis.analytics.DimensionItem;
import org.hisp.dhis.analytics.IllegalQueryException;
@@ -131,10 +133,10 @@
indicatorService.addIndicator( inA );
- deA = createDataElement( 'A' );
- deB = createDataElement( 'B' );
- deC = createDataElement( 'C' );
- deD = createDataElement( 'D' );
+ deA = createDataElement( 'A', VALUE_TYPE_INT, AGGREGATION_OPERATOR_SUM );
+ deB = createDataElement( 'B', VALUE_TYPE_INT, AGGREGATION_OPERATOR_SUM );
+ deC = createDataElement( 'C', VALUE_TYPE_INT, AGGREGATION_OPERATOR_AVERAGE );
+ deD = createDataElement( 'D', VALUE_TYPE_INT, AGGREGATION_OPERATOR_AVERAGE );
dataElementService.addDataElement( deA );
dataElementService.addDataElement( deB );
@@ -374,8 +376,8 @@
/**
* Query spans 2 partitions. Splits in 2 queries for each partition, then
- * splits in 2 queries on data elements to satisfy optimal for a total
- * of 4 queries.
+ * splits in 4 queries on data elements to satisfy optimal for a total
+ * of 8 queries, because query has 2 different aggregation types.
*/
@Test
public void planQueryA()
@@ -385,11 +387,13 @@
params.setOrganisationUnits( getList( ouA, ouB, ouC, ouD, ouE ) );
params.setPeriods( getList( createPeriod( "2000Q1" ), createPeriod( "2000Q2" ), createPeriod( "2000Q3" ), createPeriod( "2000Q4" ), createPeriod( "2001Q1" ), createPeriod( "2001Q2" ) ) );
- List<DataQueryParams> queries = queryPlanner.planQuery( params, 4, ANALYTICS_TABLE_NAME );
-
- assertEquals( 4, queries.size() );
-
- for ( DataQueryParams query : queries )
+ DataQueryGroups queryGroups = queryPlanner.planQuery( params, 4, ANALYTICS_TABLE_NAME );
+
+ assertEquals( 8, queryGroups.getAllQueries().size() );
+ assertEquals( 2, queryGroups.getSequentialQueries().size() );
+ assertEquals( 4, queryGroups.getLargestGroupSize() );
+
+ for ( DataQueryParams query : queryGroups.getAllQueries() )
{
assertTrue( samePeriodType( query.getPeriods() ) );
assertTrue( samePartition( query.getPeriods() ) );
@@ -410,11 +414,13 @@
params.setOrganisationUnits( getList( ouA, ouB, ouC, ouD, ouE ) );
params.setPeriods( getList( createPeriod( "2000Q1" ), createPeriod( "2000Q2" ), createPeriod( "2000" ), createPeriod( "200002" ), createPeriod( "200003" ), createPeriod( "200004" ) ) );
- List<DataQueryParams> queries = queryPlanner.planQuery( params, 6, ANALYTICS_TABLE_NAME );
-
- assertEquals( 6, queries.size() );
-
- for ( DataQueryParams query : queries )
+ DataQueryGroups queryGroups = queryPlanner.planQuery( params, 6, ANALYTICS_TABLE_NAME );
+
+ assertEquals( 6, queryGroups.getAllQueries().size() );
+ assertEquals( 1, queryGroups.getSequentialQueries().size() );
+ assertEquals( 6, queryGroups.getLargestGroupSize() );
+
+ for ( DataQueryParams query : queryGroups.getAllQueries() )
{
assertTrue( samePeriodType( query.getPeriods() ) );
assertTrue( samePartition( query.getPeriods() ) );
@@ -449,11 +455,13 @@
params.setOrganisationUnits( getList( ouA, ouB, ouC, ouD, ouE ) );
params.setPeriods( getList( createPeriod( "2000Q1" ), createPeriod( "2000Q2" ), createPeriod( "2000Q3" ) ) );
- List<DataQueryParams> queries = queryPlanner.planQuery( params, 6, ANALYTICS_TABLE_NAME );
-
- assertEquals( 5, queries.size() );
-
- for ( DataQueryParams query : queries )
+ DataQueryGroups queryGroups = queryPlanner.planQuery( params, 6, ANALYTICS_TABLE_NAME );
+
+ assertEquals( 5, queryGroups.getAllQueries().size() );
+ assertEquals( 1, queryGroups.getSequentialQueries().size() );
+ assertEquals( 5, queryGroups.getLargestGroupSize() );
+
+ for ( DataQueryParams query : queryGroups.getAllQueries() )
{
assertTrue( samePeriodType( query.getPeriods() ) );
assertTrue( samePartition( query.getPeriods() ) );
@@ -473,11 +481,13 @@
params.setPeriods( getList( createPeriod( "200001" ), createPeriod( "200002" ), createPeriod( "200003" ), createPeriod( "200004" ),
createPeriod( "200005" ), createPeriod( "200006" ), createPeriod( "200007" ), createPeriod( "200008" ), createPeriod( "200009" ) ) );
- List<DataQueryParams> queries = queryPlanner.planQuery( params, 4, ANALYTICS_TABLE_NAME );
-
- assertEquals( 3, queries.size() );
-
- for ( DataQueryParams query : queries )
+ DataQueryGroups queryGroups = queryPlanner.planQuery( params, 4, ANALYTICS_TABLE_NAME );
+
+ assertEquals( 3, queryGroups.getAllQueries().size() );
+ assertEquals( 2, queryGroups.getSequentialQueries().size() );
+ assertEquals( 2, queryGroups.getLargestGroupSize() );
+
+ for ( DataQueryParams query : queryGroups.getAllQueries() )
{
assertTrue( samePeriodType( query.getPeriods() ) );
assertTrue( samePartition( query.getPeriods() ) );
@@ -496,11 +506,13 @@
params.setPeriods( getList( createPeriod( "200001" ), createPeriod( "200002" ), createPeriod( "200003" ), createPeriod( "200004" ),
createPeriod( "200005" ), createPeriod( "200006" ), createPeriod( "200007" ), createPeriod( "200008" ), createPeriod( "200009" ) ) );
- List<DataQueryParams> queries = queryPlanner.planQuery( params, 4, ANALYTICS_TABLE_NAME );
-
- assertEquals( 3, queries.size() );
-
- for ( DataQueryParams query : queries )
+ DataQueryGroups queryGroups = queryPlanner.planQuery( params, 4, ANALYTICS_TABLE_NAME );
+
+ assertEquals( 3, queryGroups.getAllQueries().size() );
+ assertEquals( 2, queryGroups.getSequentialQueries().size() );
+ assertEquals( 2, queryGroups.getLargestGroupSize() );
+
+ for ( DataQueryParams query : queryGroups.getAllQueries() )
{
assertTrue( samePeriodType( query.getPeriods() ) );
assertTrue( samePartition( query.getPeriods() ) );
@@ -509,7 +521,8 @@
}
/**
- * Splits on 5 organisation units. No data elements units specified.
+ * Splits on 3 queries on organisation units for an optimal of 4 queries. No
+ * data elements specified.
*/
@Test
public void planQueryF()
@@ -519,11 +532,13 @@
params.setPeriods( getList( createPeriod( "200001" ), createPeriod( "200002" ), createPeriod( "200003" ), createPeriod( "200004" ),
createPeriod( "200005" ), createPeriod( "200006" ), createPeriod( "200007" ), createPeriod( "200008" ), createPeriod( "200009" ) ) );
- List<DataQueryParams> queries = queryPlanner.planQuery( params, 4, ANALYTICS_TABLE_NAME );
-
- assertEquals( 3, queries.size() );
-
- for ( DataQueryParams query : queries )
+ DataQueryGroups queryGroups = queryPlanner.planQuery( params, 4, ANALYTICS_TABLE_NAME );
+
+ assertEquals( 3, queryGroups.getAllQueries().size() );
+ assertEquals( 1, queryGroups.getSequentialQueries().size() );
+ assertEquals( 3, queryGroups.getLargestGroupSize() );
+
+ for ( DataQueryParams query : queryGroups.getAllQueries() )
{
assertTrue( samePeriodType( query.getPeriods() ) );
assertTrue( samePartition( query.getPeriods() ) );
@@ -546,7 +561,8 @@
/**
* Query filters span 2 partitions. Splits in 4 queries on data elements to
- * satisfy optimal for a total of 4 queries.
+ * satisfy optimal for a total of 8 queries, because query has 2 different
+ * aggregation types.
*/
@Test
public void planQueryH()
@@ -556,11 +572,13 @@
params.setOrganisationUnits( getList( ouA, ouB, ouC, ouD, ouE ) );
params.setFilterPeriods( getList( createPeriod( "2000Q1" ), createPeriod( "2000Q2" ), createPeriod( "2000Q3" ), createPeriod( "2000Q4" ), createPeriod( "2001Q1" ), createPeriod( "2001Q2" ) ) );
- List<DataQueryParams> queries = queryPlanner.planQuery( params, 4, ANALYTICS_TABLE_NAME );
+ DataQueryGroups queryGroups = queryPlanner.planQuery( params, 4, ANALYTICS_TABLE_NAME );
- assertEquals( 4, queries.size() );
+ assertEquals( 8, queryGroups.getAllQueries().size() );
+ assertEquals( 2, queryGroups.getSequentialQueries().size() );
+ assertEquals( 4, queryGroups.getLargestGroupSize() );
- for ( DataQueryParams query : queries )
+ for ( DataQueryParams query : queryGroups.getAllQueries() )
{
assertDimensionNameNotNull( query );
@@ -572,8 +590,8 @@
/**
* Query spans 3 period types. Splits in 3 queries for each period type, then
- * splits in 2 queries on data elements units to satisfy optimal for a total
- * of 6 queries.
+ * splits in 4 queries on data elements units to satisfy optimal for a total
+ * of 12 queries, because query has 2 different aggregation types.
*/
@Test
public void planQueryI()
@@ -583,11 +601,13 @@
params.setOrganisationUnits( getList( ouA, ouB, ouC, ouD, ouE ) );
params.setPeriods( getList( createPeriod( "2000Q1" ), createPeriod( "2000Q2" ), createPeriod( "2000" ), createPeriod( "200002" ), createPeriod( "200003" ), createPeriod( "200004" ) ) );
- List<DataQueryParams> queries = queryPlanner.planQuery( params, 6, ANALYTICS_TABLE_NAME );
+ DataQueryGroups queryGroups = queryPlanner.planQuery( params, 6, ANALYTICS_TABLE_NAME );
- assertEquals( 6, queries.size() );
+ assertEquals( 12, queryGroups.getAllQueries().size() );
+ assertEquals( 2, queryGroups.getSequentialQueries().size() );
+ assertEquals( 6, queryGroups.getLargestGroupSize() );
- for ( DataQueryParams query : queries )
+ for ( DataQueryParams query : queryGroups.getAllQueries() )
{
assertTrue( samePeriodType( query.getPeriods() ) );
assertTrue( samePartition( query.getPeriods() ) );
@@ -621,7 +641,7 @@
params.setOrganisationUnits( getList( ouA, ouB, ouC, ouD, ouE ) );
params.setPeriods( getList( createPeriod( "2000Q1" ), createPeriod( "2000Q2" ), createPeriod( "2000Q3" ), createPeriod( "2000Q4" ), createPeriod( "2001Q1" ), createPeriod( "2001Q2" ) ) );
- List<DataQueryParams> queries = queryPlanner.planQuery( params, 4, ANALYTICS_TABLE_NAME );
+ List<DataQueryParams> queries = queryPlanner.planQuery( params, 4, ANALYTICS_TABLE_NAME ).getAllQueries();
assertEquals( 4, queries.size() );
=== modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/Timer.java'
--- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/Timer.java 2013-03-20 13:04:15 +0000
+++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/Timer.java 2013-05-26 14:06:51 +0000
@@ -58,7 +58,7 @@
if ( !printDisabled )
{
- System.out.println( msg + ": " + time + " micros" );
+ System.out.println( "Time: " + time + " micros: " + msg );
}
return time;
