dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #28731
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 14381: Event query analytics. Using a single query instead of splitting into multiple. Fixes issue with ...
Merge authors:
Lars Helge Øverland (larshelge)
------------------------------------------------------------
revno: 14381 [merge]
committer: Lars Helge Øverland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Mon 2014-03-24 23:22:06 +0100
message:
Event query analytics. Using a single query instead of splitting into multiple. Fixes issue with paging.
modified:
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/Partitions.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryPlanner.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsService.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java
dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EventQueryPlannerTest.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/Partitions.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/Partitions.java 2014-03-18 08:10:10 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/Partitions.java 2014-03-24 21:32:04 +0000
@@ -95,13 +95,13 @@
/**
* Prunes this instance so that it retains only the partitions included in
- * the given list. No operation takes place if the given live is null.
+ * the given list. No operation takes place if the given list is null or empty.
*
* @param validPartitions list of valid partitions to retain.
*/
public Partitions prunePartitions( List<String> validPartitions )
{
- if ( validPartitions != null )
+ if ( validPartitions != null && !validPartitions.isEmpty() )
{
partitions.retainAll( validPartitions );
}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryPlanner.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryPlanner.java 2014-03-18 08:10:10 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryPlanner.java 2014-03-24 22:19:59 +0000
@@ -46,8 +46,8 @@
* Plans the given params and returns a list of params.
*
* @param params the query params.
- * @param validPartitions the list of existing database partition names, only
- * required for aggregate queries.
*/
- List<EventQueryParams> planQuery( EventQueryParams params, List<String> validPartitions );
+ List<EventQueryParams> planAggregateQuery( EventQueryParams params );
+
+ EventQueryParams planEventQuery( EventQueryParams params );
}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsService.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsService.java 2014-03-18 08:10:10 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsService.java 2014-03-24 22:19:59 +0000
@@ -125,14 +125,11 @@
// TODO order event analytics tables on execution date to avoid default
// TODO sorting in queries
- // TODO parallel processing of queries
public Grid getAggregatedEventData( EventQueryParams params )
{
queryPlanner.validate( params );
- List<String> validPartitions = analyticsManager.getAnalyticsTables( params.getProgram() );
-
Grid grid = new ListGrid();
// ---------------------------------------------------------------------
@@ -155,7 +152,7 @@
// Data
// ---------------------------------------------------------------------
- List<EventQueryParams> queries = queryPlanner.planQuery( params, validPartitions );
+ List<EventQueryParams> queries = queryPlanner.planAggregateQuery( params );
for ( EventQueryParams query : queries )
{
@@ -222,24 +219,24 @@
Timer t = new Timer().start();
- List<EventQueryParams> queries = queryPlanner.planQuery( params, null );
+ params = queryPlanner.planEventQuery( params );
- t.getSplitTime( "Planned query, got: " + queries.size() );
+ t.getSplitTime( "Planned query, got partitions: " + params.getPartitions() );
int count = 0;
- for ( EventQueryParams query : queries )
+ if ( params.getPartitions().hasAny() )
{
if ( params.isPaging() )
{
- count += analyticsManager.getEventCount( query );
+ count += analyticsManager.getEventCount( params );
}
-
- analyticsManager.getEvents( query, grid );
+
+ analyticsManager.getEvents( params, grid );
+
+ t.getTime( "Queried events, got: " + grid.getHeight() );
}
-
- t.getTime( "Queried events, got: " + grid.getHeight() );
-
+
// ---------------------------------------------------------------------
// Meta-data
// ---------------------------------------------------------------------
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java 2014-03-18 08:10:10 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java 2014-03-24 22:19:59 +0000
@@ -31,7 +31,6 @@
import static org.hisp.dhis.common.DimensionalObject.PERIOD_DIM_ID;
import java.util.ArrayList;
-import java.util.Date;
import java.util.List;
import org.apache.commons.logging.Log;
@@ -39,15 +38,16 @@
import org.hisp.dhis.analytics.DataQueryParams;
import org.hisp.dhis.analytics.Partitions;
import org.hisp.dhis.analytics.QueryPlanner;
+import org.hisp.dhis.analytics.event.EventAnalyticsManager;
import org.hisp.dhis.analytics.event.EventQueryParams;
import org.hisp.dhis.analytics.event.EventQueryPlanner;
import org.hisp.dhis.analytics.table.PartitionUtils;
import org.hisp.dhis.common.IllegalQueryException;
import org.hisp.dhis.common.ListMap;
import org.hisp.dhis.common.NameableObject;
-import org.hisp.dhis.period.Cal;
+import org.hisp.dhis.organisationunit.OrganisationUnit;
+import org.hisp.dhis.organisationunit.OrganisationUnitService;
import org.hisp.dhis.period.Period;
-import org.hisp.dhis.program.Program;
import org.springframework.beans.factory.annotation.Autowired;
/**
@@ -60,7 +60,18 @@
@Autowired
private QueryPlanner queryPlanner;
+
+ @Autowired
+ private EventAnalyticsManager analyticsManager;
+ @Autowired
+ private OrganisationUnitService organisationUnitService;
+
+ // -------------------------------------------------------------------------
+ // EventQueryPlanner implementation
+ // -------------------------------------------------------------------------
+
+ @Override
public void validate( EventQueryParams params )
throws IllegalQueryException
{
@@ -107,8 +118,11 @@
}
}
- public List<EventQueryParams> planQuery( EventQueryParams params, List<String> validPartitions )
+ @Override
+ public List<EventQueryParams> planAggregateQuery( EventQueryParams params )
{
+ List<String> validPartitions = analyticsManager.getAnalyticsTables( params.getProgram() );
+
List<EventQueryParams> queries = new ArrayList<EventQueryParams>();
List<EventQueryParams> groupedByPartition = groupByPartition( params, validPartitions );
@@ -125,63 +139,58 @@
return queries;
}
-
+
+ @Override
+ public EventQueryParams planEventQuery( EventQueryParams params )
+ {
+ List<String> validPartitions = analyticsManager.getAnalyticsTables( params.getProgram() );
+
+ String tableSuffix = "_" + params.getProgram().getUid();
+
+ if ( params.hasStartEndDate() )
+ {
+ Period queryPeriod = new Period();
+ queryPeriod.setStartDate( params.getStartDate() );
+ queryPeriod.setEndDate( params.getEndDate() );
+ params.setPartitions( PartitionUtils.getPartitions( queryPeriod, TABLE_PREFIX, tableSuffix, validPartitions ) );
+ }
+
+ for ( NameableObject object : params.getOrganisationUnits() )
+ {
+ OrganisationUnit unit = (OrganisationUnit) object;
+ unit.setLevel( organisationUnitService.getLevelOfOrganisationUnit( unit.getUid() ) );
+ }
+
+ //TODO periods, convert to start/end dates
+
+ return params;
+ }
+
+ // -------------------------------------------------------------------------
+ // Supportive methods
+ // -------------------------------------------------------------------------
+
private List<EventQueryParams> groupByPartition( EventQueryParams params, List<String> validPartitions )
{
List<EventQueryParams> queries = new ArrayList<EventQueryParams>();
- Program program = params.getProgram();
-
- String tableSuffix = "_" + program.getUid();
+ String tableSuffix = "_" + params.getProgram().getUid();
if ( params.hasStartEndDate() )
{
- if ( params.isAggregate() ) // Multiple partitions/years in one query
- {
- Period queryPeriod = new Period();
- queryPeriod.setStartDate( params.getStartDate() );
- queryPeriod.setEndDate( params.getEndDate() );
-
- EventQueryParams query = params.instance();
- query.setPartitions( PartitionUtils.getPartitions( queryPeriod, TABLE_PREFIX, tableSuffix, validPartitions ) );
-
- if ( query.getPartitions().hasAny() )
- {
- queries.add( query );
- }
- }
- else // Event query - split in one query per partition/year
- {
- Date startDate = params.getStartDate();
- Date endDate = params.getEndDate();
-
- Date currentStartDate = startDate;
- Date currentEndDate = endDate;
-
- while ( true )
- {
- if ( PartitionUtils.year( currentStartDate ) < PartitionUtils.year( endDate ) ) // Spans multiple
- {
- // Set end date to max of current year
-
- currentEndDate = PartitionUtils.maxOfYear( currentStartDate );
-
- queries.add( getQuery( params, currentStartDate, currentEndDate, program ) );
-
- // Set start date to start of next year
-
- currentStartDate = new Cal( ( PartitionUtils.year( currentStartDate ) + 1 ), 1, 1 ).time();
- }
- else
- {
- queries.add( getQuery( params, currentStartDate, endDate, program ) );
-
- break;
- }
- }
+ Period queryPeriod = new Period();
+ queryPeriod.setStartDate( params.getStartDate() );
+ queryPeriod.setEndDate( params.getEndDate() );
+
+ EventQueryParams query = params.instance();
+ query.setPartitions( PartitionUtils.getPartitions( queryPeriod, TABLE_PREFIX, tableSuffix, validPartitions ) );
+
+ if ( query.getPartitions().hasAny() )
+ {
+ queries.add( query );
}
}
- else
+ else // Aggregate only
{
ListMap<Partitions, NameableObject> partitionPeriodMap = PartitionUtils.getPartitionPeriodMap( params.getDimensionOrFilter( PERIOD_DIM_ID ), TABLE_PREFIX, tableSuffix );
@@ -196,17 +205,7 @@
return queries;
}
-
- private EventQueryParams getQuery( EventQueryParams params, Date startDate, Date endDate, Program program )
- {
- EventQueryParams query = params.instance();
- query.setStartDate( startDate );
- query.setEndDate( endDate );
- String tableName = TABLE_PREFIX + "_" + PartitionUtils.year( startDate ) + "_" + program.getUid();
- query.setPartitions( new Partitions().add( tableName ) );
- return query;
- }
-
+
private static List<EventQueryParams> convert( List<DataQueryParams> params )
{
List<EventQueryParams> eventParams = new ArrayList<EventQueryParams>();
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EventQueryPlannerTest.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EventQueryPlannerTest.java 2014-03-18 08:10:10 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EventQueryPlannerTest.java 2014-03-24 22:19:59 +0000
@@ -34,9 +34,11 @@
import java.util.List;
import org.hisp.dhis.DhisSpringTest;
+import org.hisp.dhis.analytics.Partitions;
import org.hisp.dhis.analytics.event.EventQueryParams;
import org.hisp.dhis.analytics.event.EventQueryPlanner;
import org.hisp.dhis.organisationunit.OrganisationUnit;
+import org.hisp.dhis.organisationunit.OrganisationUnitService;
import org.hisp.dhis.period.Cal;
import org.hisp.dhis.program.Program;
import org.junit.Test;
@@ -55,21 +57,27 @@
@Autowired
private EventQueryPlanner queryPlanner;
+ @Autowired
+ private OrganisationUnitService organisationUnitService;
+
@Override
public void setUpTest()
{
prA = new Program();
- prA.setUid( "programuidA" );
+ prA.setUid( "programuida" );
ouA = createOrganisationUnit( 'A' );
ouB = createOrganisationUnit( 'B' );
+ organisationUnitService.addOrganisationUnit( ouA );
+ organisationUnitService.addOrganisationUnit( ouB );
+
ouA.setLevel( 1 );
ouB.setLevel( 2 );
}
@Test
- public void testPlanQueryA()
+ public void testPlanAggregateQueryA()
{
EventQueryParams params = new EventQueryParams();
params.setProgram( prA );
@@ -77,24 +85,25 @@
params.setEndDate( new Cal( 2012, 3, 20 ).time() );
params.setOrganisationUnits( Arrays.asList( ouA ) );
- List<EventQueryParams> queries = queryPlanner.planQuery( params, null );
-
- assertEquals( 3, queries.size() );
-
- assertEquals( new Cal( 2010, 6, 1 ).time(), queries.get( 0 ).getStartDate() );
- assertEquals( new Cal( 2010, 12, 31 ).time(), queries.get( 0 ).getEndDate() );
- assertEquals( new Cal( 2011, 1, 1 ).time(), queries.get( 1 ).getStartDate() );
- assertEquals( new Cal( 2011, 12, 31 ).time(), queries.get( 1 ).getEndDate() );
- assertEquals( new Cal( 2012, 1, 1 ).time(), queries.get( 2 ).getStartDate() );
- assertEquals( new Cal( 2012, 3, 20 ).time(), queries.get( 2 ).getEndDate() );
-
- assertEquals( "analytics_event_2010_programuidA", queries.get( 0 ).getPartitions().getSinglePartition() );
- assertEquals( "analytics_event_2011_programuidA", queries.get( 1 ).getPartitions().getSinglePartition() );
- assertEquals( "analytics_event_2012_programuidA", queries.get( 2 ).getPartitions().getSinglePartition() );
+ List<EventQueryParams> queries = queryPlanner.planAggregateQuery( params );
+
+ assertEquals( 1, queries.size() );
+
+ EventQueryParams query = queries.get( 0 );
+
+ assertEquals( new Cal( 2010, 6, 1 ).time(), query.getStartDate() );
+ assertEquals( new Cal( 2012, 3, 20 ).time(), query.getEndDate() );
+
+ Partitions partitions = query.getPartitions();
+
+ assertEquals( 3, partitions.getPartitions().size() );
+ assertEquals( "analytics_event_2010_programuida", partitions.getPartitions().get( 0 ) );
+ assertEquals( "analytics_event_2011_programuida", partitions.getPartitions().get( 1 ) );
+ assertEquals( "analytics_event_2012_programuida", partitions.getPartitions().get( 2 ) );
}
@Test
- public void testPlanQueryB()
+ public void testPlanAggregateQueryB()
{
EventQueryParams params = new EventQueryParams();
params.setProgram( prA );
@@ -102,18 +111,23 @@
params.setEndDate( new Cal( 2010, 9, 20 ).time() );
params.setOrganisationUnits( Arrays.asList( ouA ) );
- List<EventQueryParams> queries = queryPlanner.planQuery( params, null );
+ List<EventQueryParams> queries = queryPlanner.planAggregateQuery( params );
assertEquals( 1, queries.size() );
+
+ EventQueryParams query = queries.get( 0 );
- assertEquals( new Cal( 2010, 3, 1 ).time(), queries.get( 0 ).getStartDate() );
- assertEquals( new Cal( 2010, 9, 20 ).time(), queries.get( 0 ).getEndDate() );
-
- assertEquals( "analytics_event_2010_programuidA", queries.get( 0 ).getPartitions().getSinglePartition() );
- }
+ assertEquals( new Cal( 2010, 3, 1 ).time(), query.getStartDate() );
+ assertEquals( new Cal( 2010, 9, 20 ).time(), query.getEndDate() );
+
+ Partitions partitions = query.getPartitions();
+
+ assertEquals( 1, partitions.getPartitions().size() );
+ assertEquals( "analytics_event_2010_programuida", partitions.getSinglePartition() );
+ }
@Test
- public void testPlanQueryC()
+ public void testPlanAggregateQueryC()
{
EventQueryParams params = new EventQueryParams();
params.setProgram( prA );
@@ -121,8 +135,52 @@
params.setEndDate( new Cal( 2012, 3, 20 ).time() );
params.setOrganisationUnits( Arrays.asList( ouA, ouB ) );
- List<EventQueryParams> queries = queryPlanner.planQuery( params, null );
-
- assertEquals( 6, queries.size() );
+ List<EventQueryParams> queries = queryPlanner.planAggregateQuery( params );
+
+ assertEquals( 2, queries.size() );
+ assertEquals( ouA, queries.get( 0 ).getOrganisationUnits().get( 0 ) );
+ assertEquals( ouB, queries.get( 1 ).getOrganisationUnits().get( 0 ) );
+ }
+
+ @Test
+ public void testPlanEventQueryA()
+ {
+ EventQueryParams params = new EventQueryParams();
+ params.setProgram( prA );
+ params.setStartDate( new Cal( 2010, 6, 1 ).time() );
+ params.setEndDate( new Cal( 2012, 3, 20 ).time() );
+ params.setOrganisationUnits( Arrays.asList( ouA ) );
+
+ params = queryPlanner.planEventQuery( params );
+
+ assertEquals( new Cal( 2010, 6, 1 ).time(), params.getStartDate() );
+ assertEquals( new Cal( 2012, 3, 20 ).time(), params.getEndDate() );
+
+ Partitions partitions = params.getPartitions();
+
+ assertEquals( 3, partitions.getPartitions().size() );
+ assertEquals( "analytics_event_2010_programuida", partitions.getPartitions().get( 0 ) );
+ assertEquals( "analytics_event_2011_programuida", partitions.getPartitions().get( 1 ) );
+ assertEquals( "analytics_event_2012_programuida", partitions.getPartitions().get( 2 ) );
+ }
+
+ @Test
+ public void testPlanEventQueryB()
+ {
+ EventQueryParams params = new EventQueryParams();
+ params.setProgram( prA );
+ params.setStartDate( new Cal( 2010, 3, 1 ).time() );
+ params.setEndDate( new Cal( 2010, 9, 20 ).time() );
+ params.setOrganisationUnits( Arrays.asList( ouA ) );
+
+ params = queryPlanner.planEventQuery( params );
+
+ assertEquals( new Cal( 2010, 3, 1 ).time(), params.getStartDate() );
+ assertEquals( new Cal( 2010, 9, 20 ).time(), params.getEndDate() );
+
+ Partitions partitions = params.getPartitions();
+
+ assertEquals( 1, partitions.getPartitions().size() );
+ assertEquals( "analytics_event_2010_programuida", partitions.getSinglePartition() );
}
}
