dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #25612
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 12759: Event analytics, improved partitioning
Merge authors:
Lars Helge Øverland (larshelge)
------------------------------------------------------------
revno: 12759 [merge]
committer: Lars Helge Øverland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Wed 2013-10-16 22:44:52 +0200
message:
Event analytics, improved partitioning
modified:
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/PartitionUtils.java
dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/PartitionUtilsTest.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java 2013-10-16 20:12:01 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java 2013-10-16 20:44:25 +0000
@@ -46,6 +46,7 @@
import org.hisp.dhis.common.ListMap;
import org.hisp.dhis.common.NameableObject;
import org.hisp.dhis.period.Cal;
+import org.hisp.dhis.period.Period;
import org.hisp.dhis.program.Program;
import org.springframework.beans.factory.annotation.Autowired;
@@ -132,41 +133,54 @@
List<EventQueryParams> queries = new ArrayList<EventQueryParams>();
Program program = params.getProgram();
+
+ String tableSuffix = "_" + program.getUid();
if ( params.hasStartEndDate() )
{
- Date startDate = params.getStartDate();
- Date endDate = params.getEndDate();
-
- Date currentStartDate = startDate;
- Date currentEndDate = endDate;
-
- while ( true )
- {
- if ( year( currentStartDate ) < year( endDate ) ) // Spans multiple
- {
- // Set end date to max of current year
-
- currentEndDate = maxOfYear( currentStartDate );
-
- queries.add( getQuery( params, currentStartDate, currentEndDate, program ) );
-
- // Set start date to start of next year
-
- currentStartDate = new Cal( ( year( currentStartDate ) + 1 ), 1, 1 ).time();
- }
- else
- {
- queries.add( getQuery( params, currentStartDate, endDate, program ) );
-
- break;
+ if ( params.isAggregate() ) // Multiple partitions/years in one query
+ {
+ Period queryPeriod = new Period();
+ queryPeriod.setStartDate( params.getStartDate() );
+ queryPeriod.setEndDate( params.getEndDate() );
+
+ EventQueryParams query = params.instance();
+ query.setPartitions( PartitionUtils.getPartitions( queryPeriod, TABLE_PREFIX, tableSuffix ) );
+ queries.add( query );
+ }
+ else // Event query - split in one query per partition/year
+ {
+ Date startDate = params.getStartDate();
+ Date endDate = params.getEndDate();
+
+ Date currentStartDate = startDate;
+ Date currentEndDate = endDate;
+
+ while ( true )
+ {
+ if ( PartitionUtils.year( currentStartDate ) < PartitionUtils.year( endDate ) ) // Spans multiple
+ {
+ // Set end date to max of current year
+
+ currentEndDate = PartitionUtils.maxOfYear( currentStartDate );
+
+ queries.add( getQuery( params, currentStartDate, currentEndDate, program ) );
+
+ // Set start date to start of next year
+
+ currentStartDate = new Cal( ( PartitionUtils.year( currentStartDate ) + 1 ), 1, 1 ).time();
+ }
+ else
+ {
+ queries.add( getQuery( params, currentStartDate, endDate, program ) );
+
+ break;
+ }
}
}
}
else
{
- String tableSuffix = "_" + program.getUid();
-
ListMap<Partitions, NameableObject> partitionPeriodMap = PartitionUtils.getPartitionPeriodMap( params.getDimensionOrFilter( PERIOD_DIM_ID ), TABLE_PREFIX, tableSuffix );
for ( Partitions partitions : partitionPeriodMap.keySet() )
@@ -186,21 +200,11 @@
EventQueryParams query = params.instance();
query.setStartDate( startDate );
query.setEndDate( endDate );
- String tableName = TABLE_PREFIX + "_" + year( startDate ) + "_" + program.getUid();
+ String tableName = TABLE_PREFIX + "_" + PartitionUtils.year( startDate ) + "_" + program.getUid();
query.setPartitions( new Partitions().add( tableName ) );
return query;
}
- private static int year( Date date )
- {
- return new Cal( date ).getYear();
- }
-
- private static Date maxOfYear( Date date )
- {
- return new Cal( year( date ), 12, 31 ).time();
- }
-
private static List<EventQueryParams> convert( List<DataQueryParams> params )
{
List<EventQueryParams> eventParams = new ArrayList<EventQueryParams>();
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/PartitionUtils.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/PartitionUtils.java 2013-10-16 19:25:28 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/PartitionUtils.java 2013-10-16 20:37:25 +0000
@@ -39,6 +39,7 @@
import org.hisp.dhis.analytics.Partitions;
import org.hisp.dhis.common.ListMap;
import org.hisp.dhis.common.NameableObject;
+import org.hisp.dhis.period.Cal;
import org.hisp.dhis.period.Period;
import org.hisp.dhis.period.YearlyPeriodType;
@@ -66,7 +67,6 @@
return periods;
}
- //TODO allow periods spanning more than two years
//TODO optimize by including required filter periods only
public static Partitions getPartitions( Period period, String tablePrefix, String tableSuffix )
@@ -76,14 +76,13 @@
Partitions partitions = new Partitions();
- Period startYear = PERIODTYPE.createPeriod( period.getStartDate() );
- Period endYear = PERIODTYPE.createPeriod( period.getEndDate() );
-
- partitions.add( tablePrefix + SEP + startYear.getIsoDate() + tableSuffix );
-
- if ( !startYear.equals( endYear ) )
+ int startYear = year( period.getStartDate() );
+ int endYear = year( period.getEndDate() );
+
+ while ( startYear <= endYear )
{
- partitions.add( tablePrefix + SEP + endYear.getIsoDate() + tableSuffix );
+ partitions.add( tablePrefix + SEP + startYear + tableSuffix );
+ startYear++;
}
return partitions;
@@ -129,4 +128,20 @@
return map;
}
+
+ /**
+ * Returns the year of the given date.
+ */
+ public static int year( Date date )
+ {
+ return new Cal( date ).getYear();
+ }
+
+ /**
+ * Returns the max date within the year of the given date.
+ */
+ public static Date maxOfYear( Date date )
+ {
+ return new Cal( year( date ), 12, 31 ).time();
+ }
}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/PartitionUtilsTest.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/PartitionUtilsTest.java 2013-10-16 19:21:01 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/PartitionUtilsTest.java 2013-10-16 20:37:25 +0000
@@ -40,6 +40,8 @@
import org.hisp.dhis.analytics.Partitions;
import org.hisp.dhis.common.ListMap;
import org.hisp.dhis.common.NameableObject;
+import org.hisp.dhis.period.Cal;
+import org.hisp.dhis.period.Period;
import org.junit.Test;
/**
@@ -60,7 +62,8 @@
assertEquals( new Partitions().add( TBL + "_2000" ).add( TBL + "_2001" ), PartitionUtils.getPartitions( createPeriod( "2000July" ), TBL, null ) );
assertEquals( new Partitions().add( TBL + "_2001" ).add( TBL + "_2002" ), PartitionUtils.getPartitions( createPeriod( "2001April" ), TBL, null ) );
}
-
+
+ @Test
public void getGetPartitionsMultiplePeriods()
{
List<NameableObject> periods = new ArrayList<NameableObject>();
@@ -70,6 +73,26 @@
assertEquals( new Partitions().add( TBL + "_2000" ).add( TBL + "_2001" ), PartitionUtils.getPartitions( periods, TBL, null ) );
}
+
+ @Test
+ public void getGetPartitionsLongPeriods()
+ {
+ Period period = new Period();
+ period.setStartDate( new Cal( 2008, 3, 1 ).time() );
+ period.setEndDate( new Cal( 2011, 7, 1 ).time() );
+
+ Partitions expected = new Partitions().add( TBL + "_2008" ).add( TBL + "_2009" ).add( TBL + "_2010" ).add( TBL + "_2011" );
+
+ assertEquals( expected, PartitionUtils.getPartitions( period, TBL, null ) );
+
+ period = new Period();
+ period.setStartDate( new Cal( 2009, 8, 1 ).time() );
+ period.setEndDate( new Cal( 2010, 2, 1 ).time() );
+
+ expected = new Partitions().add( TBL + "_2009" ).add( TBL + "_2010" );
+
+ assertEquals( expected, PartitionUtils.getPartitions( period, TBL, null ) );
+ }
@Test
public void testGetTablePeriodMapA()
