dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #21009
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 9839: Analytics, fine-tuning
Merge authors:
Lars Helge Øverland (larshelge)
------------------------------------------------------------
revno: 9839 [merge]
committer: Lars Helge Øverland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Tue 2013-02-19 10:59:46 +0100
message:
Analytics, fine-tuning
modified:
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java
dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java
dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/MathUtils.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java 2013-02-14 13:25:21 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java 2013-02-19 09:29:27 +0000
@@ -97,6 +97,7 @@
private static final Log log = LogFactory.getLog( DefaultAnalyticsService.class );
private static final String VALUE_HEADER_NAME = "Value";
+ private static final int MAX_QUERIES = 6;
//TODO completeness
//TODO make sure data x dims are successive
@@ -142,6 +143,7 @@
params.conform();
+
// ---------------------------------------------------------------------
// Headers and meta-data
// ---------------------------------------------------------------------
@@ -279,7 +281,7 @@
Timer t = new Timer().start();
- int optimalQueries = MathUtils.getWithin( SystemUtils.getCpuCores(), 1, 6 );
+ int optimalQueries = MathUtils.getWithin( SystemUtils.getCpuCores(), 1, MAX_QUERIES );
List<DataQueryParams> queries = queryPlanner.planQuery( params, optimalQueries, tableName );
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java 2013-02-07 12:24:13 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java 2013-02-19 09:59:26 +0000
@@ -164,9 +164,11 @@
}
// ---------------------------------------------------------------------
- // Group by organisation unit
+ // Group by data element
// ---------------------------------------------------------------------
+ //TODO change order
+
queries = splitByDimensionOrFilter( queries, DataQueryParams.ORGUNIT_DIM_ID, optimalQueries );
if ( queries.size() >= optimalQueries )
@@ -175,10 +177,12 @@
}
// ---------------------------------------------------------------------
- // Group by data element
+ // Group by organiastion unit
// ---------------------------------------------------------------------
- return splitByDimensionOrFilter( queries, DataQueryParams.DATAELEMENT_DIM_ID, optimalQueries );
+ queries = splitByDimensionOrFilter( queries, DataQueryParams.DATAELEMENT_DIM_ID, optimalQueries );
+
+ return queries;
}
public boolean canQueryFromDataMart( DataQueryParams params )
@@ -195,7 +199,7 @@
*/
private List<DataQueryParams> splitByDimensionOrFilter( List<DataQueryParams> queries, String dimension, int optimalQueries )
{
- int optimalForSubQuery = MathUtils.divideToCeil( optimalQueries, queries.size() );
+ int optimalForSubQuery = MathUtils.divideToFloor( optimalQueries, queries.size() );
List<DataQueryParams> subQueries = new ArrayList<DataQueryParams>();
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java 2013-01-25 20:45:07 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java 2013-02-19 09:29:27 +0000
@@ -183,23 +183,6 @@
}
}
- private void vacuumTables( List<String> tables )
- {
- List<List<String>> tablePages = new PaginatedList<String>( tables ).setPageSize( getProcessNo() ).getPages();
-
- for ( List<String> tablePage : tablePages )
- {
- List<Future<?>> futures = new ArrayList<Future<?>>();
-
- for ( String table : tablePage )
- {
- futures.add( tableManager.vacuumTableAsync( table ) );
- }
-
- ConcurrentUtils.waitForCompletion( futures );
- }
- }
-
private void createIndexes( List<String> tables )
{
for ( String table : tables )
@@ -216,6 +199,23 @@
ConcurrentUtils.waitForCompletion( futures );
}
}
+
+ private void vacuumTables( List<String> tables )
+ {
+ List<List<String>> tablePages = new PaginatedList<String>( tables ).setPageSize( getProcessNo() ).getPages();
+
+ for ( List<String> tablePage : tablePages )
+ {
+ List<Future<?>> futures = new ArrayList<Future<?>>();
+
+ for ( String table : tablePage )
+ {
+ futures.add( tableManager.vacuumTableAsync( table ) );
+ }
+
+ ConcurrentUtils.waitForCompletion( futures );
+ }
+ }
private void swapTables( List<String> tables )
{
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java 2013-01-27 11:43:58 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java 2013-02-19 09:29:27 +0000
@@ -68,6 +68,9 @@
// Implementation
// -------------------------------------------------------------------------
+ //TODO create stack with index tasks for partition and column
+ // run continuously from stack until empty
+
public String getTableName()
{
return "analytics";
@@ -194,10 +197,10 @@
}
String[] de = { "de", "character(11) not null", "de.uid" };
- String[] co = { "coc", "character(11) not null", "coc.uid" };
+ String[] coc = { "coc", "character(11) not null", "coc.uid" };
String[] level = { "level", "integer", "ous.level" };
- columns.addAll( Arrays.asList( de, co, level ) );
+ columns.addAll( Arrays.asList( de, coc, level ) );
return columns;
}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java 2013-02-01 14:57:34 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java 2013-02-19 09:59:26 +0000
@@ -359,7 +359,7 @@
params.setOrganisationUnits( getList( ouA, ouB, ouC, ouD, ouE ) );
params.setPeriods( getList( createPeriod( "2000Q1" ), createPeriod( "2000Q2" ), createPeriod( "2000" ), createPeriod( "200002" ), createPeriod( "200003" ), createPeriod( "200004" ) ) );
- List<DataQueryParams> queries = queryPlanner.planQuery( params, 4, ANALYTICS_TABLE_NAME );
+ List<DataQueryParams> queries = queryPlanner.planQuery( params, 6, ANALYTICS_TABLE_NAME );
assertEquals( 6, queries.size() );
@@ -398,7 +398,7 @@
params.setOrganisationUnits( getList( ouA, ouB, ouC, ouD, ouE ) );
params.setPeriods( getList( createPeriod( "2000Q1" ), createPeriod( "2000Q2" ), createPeriod( "2000Q3" ) ) );
- List<DataQueryParams> queries = queryPlanner.planQuery( params, 4, ANALYTICS_TABLE_NAME );
+ List<DataQueryParams> queries = queryPlanner.planQuery( params, 6, ANALYTICS_TABLE_NAME );
assertEquals( 5, queries.size() );
@@ -531,7 +531,7 @@
params.setOrganisationUnits( getList( ouA, ouB, ouC, ouD, ouE ) );
params.setPeriods( getList( createPeriod( "2000Q1" ), createPeriod( "2000Q2" ), createPeriod( "2000" ), createPeriod( "200002" ), createPeriod( "200003" ), createPeriod( "200004" ) ) );
- List<DataQueryParams> queries = queryPlanner.planQuery( params, 4, ANALYTICS_TABLE_NAME );
+ List<DataQueryParams> queries = queryPlanner.planQuery( params, 6, ANALYTICS_TABLE_NAME );
assertEquals( 6, queries.size() );
=== modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/MathUtils.java'
--- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/MathUtils.java 2012-12-18 00:49:03 +0000
+++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/MathUtils.java 2013-02-19 09:59:26 +0000
@@ -445,4 +445,18 @@
return result.intValue();
}
+
+ /**
+ * Performs a division and rounds downwards to the next integer.
+ *
+ * @param numerator the numerator.
+ * @param denominator the denominator.
+ * @return an integer value.
+ */
+ public static int divideToFloor( int numerator, int denominator )
+ {
+ Double result = Math.floor( (double) numerator / denominator );
+
+ return result.intValue();
+ }
}
