dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #20461
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 9342: Improved routine, docs
------------------------------------------------------------
revno: 9342
committer: Lars Helge Øverland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Tue 2012-12-18 01:03:12 +0100
message:
Improved routine, docs
modified:
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java
dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java
dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/MathUtils.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java 2012-12-17 16:58:29 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java 2012-12-18 00:03:12 +0000
@@ -90,7 +90,7 @@
{
Timer t = new Timer().start();
- List<DataQueryParams> queries = queryPlanner.planQuery( params, 6 );
+ List<DataQueryParams> queries = queryPlanner.planQuery( params, 4 );
t.getTime( "Planned query" );
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java 2012-12-17 16:58:29 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java 2012-12-18 00:03:12 +0000
@@ -38,6 +38,7 @@
import org.hisp.dhis.organisationunit.OrganisationUnitService;
import org.hisp.dhis.period.PeriodType;
import org.hisp.dhis.system.util.ListMap;
+import org.hisp.dhis.system.util.MathUtils;
import org.hisp.dhis.system.util.PaginatedList;
import org.springframework.beans.factory.annotation.Autowired;
@@ -81,13 +82,29 @@
}
}
- // ---------------------------------------------------------------------
- // Return if number of queries are equal or larger than optimal
- // ---------------------------------------------------------------------
-
- boolean satisfied = queries.size() >= optimalQueries;
-
- if ( satisfied )
+ if ( queries.size() >= optimalQueries )
+ {
+ return queries;
+ }
+
+ // ---------------------------------------------------------------------
+ // Group by organisation unit
+ // ---------------------------------------------------------------------
+
+ queries = splitByDimension( queries, DataQueryParams.ORGUNIT_DIM_ID, optimalQueries );
+
+ if ( queries.size() >= optimalQueries )
+ {
+ return queries;
+ }
+
+ // ---------------------------------------------------------------------
+ // Group by data element
+ // ---------------------------------------------------------------------
+
+ queries = splitByDimension( queries, DataQueryParams.DATAELEMENT_DIM_ID, optimalQueries );
+
+ if ( queries.size() >= optimalQueries )
{
return queries;
}
@@ -95,30 +112,8 @@
// ---------------------------------------------------------------------
// Group by dimensions
// ---------------------------------------------------------------------
-
- List<DataQueryParams> subQueries = new ArrayList<DataQueryParams>();
-
- Double pages = Math.ceil( (double) optimalQueries / queries.size() );
-
- int optimal = pages.intValue();
-
- for ( DataQueryParams query : queries )
- {
- String dimension = getPartitionDimension( query, optimal );
-
- List<String> partitionValues = params.getDimension( dimension );
-
- List<List<String>> partitionValuePages = new PaginatedList<String>( partitionValues ).setNumberOfPages( optimal ).getPages();
-
- for ( List<String> valuePage : partitionValuePages )
- {
- DataQueryParams subQuery = new DataQueryParams( query );
- subQuery.setDimension( dimension, valuePage );
- subQueries.add( subQuery );
- }
- }
-
- return subQueries;
+
+ return splitByBestDimension( queries, optimalQueries );
}
public String getPartitionDimension( DataQueryParams params, int optimalQueries )
@@ -146,6 +141,63 @@
// -------------------------------------------------------------------------
/**
+ * Splits the given list of queries in sub queries on the given dimension.
+ */
+ private List<DataQueryParams> splitByDimension( List<DataQueryParams> queries, String dimension, int optimalQueries )
+ {
+ int pageNo = MathUtils.divideToCeil( optimalQueries, queries.size() );
+
+ List<DataQueryParams> subQueries = new ArrayList<DataQueryParams>();
+
+ for ( DataQueryParams query : queries )
+ {
+ List<String> values = query.getDimension( dimension );
+
+ List<List<String>> valuePages = new PaginatedList<String>( values ).setNumberOfPages( pageNo ).getPages();
+
+ for ( List<String> valuePage : valuePages )
+ {
+ DataQueryParams subQuery = new DataQueryParams( query );
+ subQuery.setDimension( dimension, valuePage );
+ subQueries.add( subQuery );
+ }
+ }
+
+ return subQueries;
+ }
+
+ /**
+ * Splits the given list of queries in sub queries on the most favorable
+ * dimension. This is determined by first checking if any dimensions will
+ * satisfy the optimal number of queries, if not the dimension with most
+ * options is selected.
+ */
+ private List<DataQueryParams> splitByBestDimension( List<DataQueryParams> queries, int optimalQueries )
+ {
+ int pageNo = MathUtils.divideToCeil( optimalQueries, queries.size() );
+
+ List<DataQueryParams> subQueries = new ArrayList<DataQueryParams>();
+
+ for ( DataQueryParams query : queries )
+ {
+ String dimension = getPartitionDimension( query, pageNo );
+
+ List<String> values = query.getDimension( dimension );
+
+ List<List<String>> valuePages = new PaginatedList<String>( values ).setNumberOfPages( pageNo ).getPages();
+
+ for ( List<String> valuePage : valuePages )
+ {
+ DataQueryParams subQuery = new DataQueryParams( query );
+ subQuery.setDimension( dimension, valuePage );
+ subQueries.add( subQuery );
+ }
+ }
+
+ return subQueries;
+ }
+
+ /**
* Groups the given query into sub queries based on its periods and which
* partition it should be executed against. Sets the partition table name on
* each query.
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java 2012-12-17 16:58:29 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java 2012-12-18 00:03:12 +0000
@@ -96,7 +96,7 @@
log.info( sql );
- SqlRowSet rowSet = jdbcTemplate.queryForRowSet( sql.toLowerCase() );
+ SqlRowSet rowSet = jdbcTemplate.queryForRowSet( sql );
Map<String, Double> map = new HashMap<String, Double>();
@@ -112,7 +112,7 @@
key.deleteCharAt( key.length() - SEP.length() );
Double value = rowSet.getDouble( DataQueryParams.VALUE_ID );
-
+
map.put( key.toString(), value );
}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java 2012-12-17 16:58:29 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java 2012-12-18 00:03:12 +0000
@@ -104,8 +104,8 @@
/**
* Query spans 2 partitions. Splits in 2 queries for each partition, then
- * splits in 2 queries on data elements to satisfy optimal for a total of 4
- * queries.
+ * splits in 2 queries on organisation units to satisfy optimal for a total
+ * of 4 queries.
*/
@Test
public void planQueryA()
@@ -128,8 +128,8 @@
/**
* Query spans 3 period types. Splits in 3 queries for each period type, then
- * splits in 2 queries on data elements to satisfy optimal for a total of 6
- * queries.
+ * splits in 2 queries on organisation units to satisfy optimal for a total
+ * of 6 queries.
*/
@Test
public void planQueryB()
@@ -152,8 +152,8 @@
/**
* Query spans 3 organisation unit levels. Splits in 3 queries for each level,
- * then splits in 2 queries on data elements to satisfy optimal for a total
- * of 6 queries.
+ * then splits in 2 queries on organisation units to satisfy optimal for a total
+ * of 5 queries, as there are only 5 organisation units in total.
*/
@Test
public void planQueryC()
@@ -179,7 +179,7 @@
List<DataQueryParams> queries = queryPlanner.planQuery( params, 4 );
- assertEquals( 6, queries.size() );
+ assertEquals( 5, queries.size() );
for ( DataQueryParams query : queries )
{
@@ -189,7 +189,8 @@
}
/**
- * Splits on largest dimension which is period.
+ * Splits on best dimension. Ignores organisation units, splits on 3 data elements,
+ * then splits in 2 queries on periods.
*/
@Test
public void planQueryD()
@@ -201,7 +202,7 @@
List<DataQueryParams> queries = queryPlanner.planQuery( params, 6 );
- assertEquals( 5, queries.size() );
+ assertEquals( 6, queries.size() );
for ( DataQueryParams query : queries )
{
=== modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/MathUtils.java'
--- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/MathUtils.java 2012-10-18 19:13:26 +0000
+++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/MathUtils.java 2012-12-18 00:03:12 +0000
@@ -149,7 +149,7 @@
* @param significantFigures
* @return
*/
- public static String roundToString(double value, int significantFigures)
+ public static String roundToString( double value, int significantFigures )
{
MathContext mc = new MathContext(significantFigures);
BigDecimal num = new BigDecimal(value);
@@ -415,4 +415,18 @@
double deviation = stdDev * stdDevFactor;
return average + deviation;
}
+
+ /**
+ * Performs a division and rounds upwards to the next integer.
+ *
+ * @param numerator the numerator.
+ * @param denominator the denominator.
+ * @return an integer value.
+ */
+ public static int divideToCeil( int numerator, int denominator )
+ {
+ Double result = Math.ceil( (double) numerator / denominator );
+
+ return result.intValue();
+ }
}
