dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #24247
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 11849: Analytics, supporting financial period type aggregation properly
------------------------------------------------------------
revno: 11849
committer: Lars Helge Øverland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Sat 2013-08-31 20:07:17 +0200
message:
Analytics, supporting financial period type aggregation properly
added:
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/Partitions.java
modified:
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/PartitionUtils.java
dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java
dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/PartitionUtilsTest.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java 2013-08-23 16:05:01 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java 2013-08-31 18:07:17 +0000
@@ -119,10 +119,8 @@
// Transient properties
// -------------------------------------------------------------------------
- private transient String tableName;
+ private transient Partitions partitions;
- private ListMap<String, NameableObject> tableNamePeriodMap;
-
private transient String periodType;
private transient PeriodType dataPeriodType;
@@ -144,11 +142,10 @@
this.aggregationType = params.getAggregationType();
this.measureCriteria = params.getMeasureCriteria();
- this.tableName = params.getTableName();
+ this.partitions = params.getPartitions();
this.periodType = params.getPeriodType();
this.dataPeriodType = params.getDataPeriodType();
this.skipPartitioning = params.isSkipPartitioning();
- this.tableNamePeriodMap = params.getTableNamePeriodMap();
}
// -------------------------------------------------------------------------
@@ -188,39 +185,11 @@
* If true it means that a period filter exists and that the periods span
* multiple years.
*/
- public boolean filterSpansMultiplePartitions()
- {
- return tableNamePeriodMap != null && tableNamePeriodMap.size() > 1;
- }
-
- /**
- * If the filters of this query spans more than partition, this method will
- * return a list of queries with a query for each partition, generated from
- * this query, where the table name and filter period items are set according
- * to the relevant partition.
- */
- public List<DataQueryParams> getPartitionFilterParams()
- {
- List<DataQueryParams> filters = new ArrayList<DataQueryParams>();
-
- if ( !filterSpansMultiplePartitions() )
- {
- return filters;
- }
-
- for ( String tableName : tableNamePeriodMap.keySet() )
- {
- List<NameableObject> periods = tableNamePeriodMap.get( tableName );
-
- DataQueryParams params = new DataQueryParams( this );
- params.setTableName( tableName );
- params.updateFilterOptions( PERIOD_DIM_ID, periods );
- filters.add( params );
- }
-
- return filters;
- }
-
+ public boolean spansMultiplePartitions()
+ {
+ return partitions != null && partitions.isMultiple();
+ }
+
/**
* Creates a mapping between dimension identifiers and filter dimensions. Filters
* are guaranteed not to be null.
@@ -1058,24 +1027,14 @@
// Get and set methods for transient properties
// -------------------------------------------------------------------------
- public String getTableName()
- {
- return tableName;
- }
-
- public void setTableName( String tableName )
- {
- this.tableName = tableName;
- }
-
- public ListMap<String, NameableObject> getTableNamePeriodMap()
- {
- return tableNamePeriodMap;
- }
-
- public void setTableNamePeriodMap( ListMap<String, NameableObject> tableNamePeriodMap )
- {
- this.tableNamePeriodMap = tableNamePeriodMap;
+ public Partitions getPartitions()
+ {
+ return partitions;
+ }
+
+ public void setPartitions( Partitions partitions )
+ {
+ this.partitions = partitions;
}
public String getPeriodType()
=== added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/Partitions.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/Partitions.java 1970-01-01 00:00:00 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/Partitions.java 2013-08-31 18:07:17 +0000
@@ -0,0 +1,79 @@
+package org.hisp.dhis.analytics;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class Partitions
+{
+ private List<String> partitions = new ArrayList<String>();
+
+ public Partitions()
+ {
+ }
+
+ public Partitions( List<String> partitions )
+ {
+ this.partitions = partitions;
+ }
+
+ public Partitions add( String partition )
+ {
+ partitions.add( partition );
+ return this;
+ }
+
+ public boolean isMultiple()
+ {
+ return partitions != null && partitions.size() > 1;
+ }
+
+ public String getSinglePartition()
+ {
+ return partitions.get( 0 );
+ }
+
+ public List<String> getPartitions()
+ {
+ return partitions;
+ }
+
+ public void setPartitions( List<String> partitions )
+ {
+ this.partitions = partitions;
+ }
+
+ @Override
+ public String toString()
+ {
+ return partitions.toString();
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return partitions.hashCode();
+ }
+
+ @Override
+ public boolean equals( Object object )
+ {
+ if ( this == object )
+ {
+ return true;
+ }
+
+ if ( object == null )
+ {
+ return false;
+ }
+
+ if ( getClass() != object.getClass() )
+ {
+ return false;
+ }
+
+ Partitions other = (Partitions) object;
+
+ return partitions.equals( other.partitions );
+ }
+}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java 2013-08-29 18:14:51 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java 2013-08-31 18:07:17 +0000
@@ -54,6 +54,7 @@
import org.hisp.dhis.analytics.DataQueryGroups;
import org.hisp.dhis.analytics.DataQueryParams;
import org.hisp.dhis.analytics.IllegalQueryException;
+import org.hisp.dhis.analytics.Partitions;
import org.hisp.dhis.analytics.QueryPlanner;
import org.hisp.dhis.analytics.table.PartitionUtils;
import org.hisp.dhis.common.BaseDimensionalObject;
@@ -224,7 +225,7 @@
for ( DataQueryParams byDataPeriodType : groupedByDataPeriodType )
{
- byDataPeriodType.setTableName( byPartition.getTableName() );
+ byDataPeriodType.setPartitions( byPartition.getPartitions() );
byDataPeriodType.setPeriodType( byPeriodType.getPeriodType() );
byDataPeriodType.setAggregationType( byAggregationType.getAggregationType() );
@@ -233,7 +234,7 @@
}
else
{
- byAggregationType.setTableName( byPartition.getTableName() );
+ byAggregationType.setPartitions( byPartition.getPartitions() );
byAggregationType.setPeriodType( byPeriodType.getPeriodType() );
queries.add( byAggregationType );
@@ -345,28 +346,25 @@
if ( params.isSkipPartitioning() )
{
- params.setTableName( tableName );
+ params.setPartitions( new Partitions().add( tableName ) );
queries.add( params );
}
else if ( params.getPeriods() != null && !params.getPeriods().isEmpty() )
{
- ListMap<String, NameableObject> tableNamePeriodMap = PartitionUtils.getTableNamePeriodMap( params.getPeriods(), tableName );
+ ListMap<Partitions, NameableObject> partitionPeriodMap = PartitionUtils.getPartitionPeriodMap( params.getPeriods(), tableName );
- for ( String table : tableNamePeriodMap.keySet() )
+ for ( Partitions partitions : partitionPeriodMap.keySet() )
{
DataQueryParams query = new DataQueryParams( params );
- query.setPeriods( tableNamePeriodMap.get( table ) );
- query.setTableName( table );
+ query.setPeriods( partitionPeriodMap.get( partitions ) );
+ query.setPartitions( partitions );
queries.add( query );
}
}
else if ( params.getFilterPeriods() != null && !params.getFilterPeriods().isEmpty() )
{
- ListMap<String, NameableObject> tableNamePeriodMap = PartitionUtils.getTableNamePeriodMap( params.getFilterPeriods(), tableName );
-
- DataQueryParams query = new DataQueryParams( params );
- query.setTableNamePeriodMap( tableNamePeriodMap );
- query.setTableName( tableNamePeriodMap.keySet().iterator().next() );
+ DataQueryParams query = new DataQueryParams( params );
+ query.setPartitions( PartitionUtils.getPartitions( params.getFilterPeriods(), tableName ) );
queries.add( query );
}
else
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java 2013-08-23 16:05:01 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java 2013-08-31 18:07:17 +0000
@@ -103,13 +103,13 @@
String sql = getSelectClause( params );
- if ( params.filterSpansMultiplePartitions() )
+ if ( params.spansMultiplePartitions() )
{
sql += getFromWhereClauseMultiplePartitionFilters( params );
}
else
{
- sql += getFromWhereClause( params );
+ sql += getFromWhereClause( params, params.getPartitions().getSinglePartition() );
}
sql += getGroupByClause( params );
@@ -214,9 +214,9 @@
{
String sql = "from (";
- for ( DataQueryParams filterParams : params.getPartitionFilterParams() )
+ for ( String partition : params.getPartitions().getPartitions() )
{
- sql += "select " + getCommaDelimitedString( filterParams.getQueryDimensions() ) + ", ";
+ sql += "select " + getCommaDelimitedString( params.getQueryDimensions() ) + ", ";
if ( params.isAggregationType( AVERAGE_INT ) )
{
@@ -231,7 +231,7 @@
sql += "value";
}
- sql += " " + getFromWhereClause( filterParams );
+ sql += " " + getFromWhereClause( params, partition );
sql += "union all ";
}
@@ -244,11 +244,11 @@
/**
* Generates the from clause of the query SQL.
*/
- private String getFromWhereClause( DataQueryParams params )
+ private String getFromWhereClause( DataQueryParams params, String partition )
{
SqlHelper sqlHelper = new SqlHelper();
- String sql = "from " + params.getTableName() + " ";
+ String sql = "from " + partition + " ";
for ( DimensionalObject dim : params.getQueryDimensions() )
{
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/PartitionUtils.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/PartitionUtils.java 2013-08-23 16:05:01 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/PartitionUtils.java 2013-08-31 18:07:17 +0000
@@ -30,8 +30,11 @@
import java.util.ArrayList;
import java.util.Date;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
+import org.hisp.dhis.analytics.Partitions;
import org.hisp.dhis.common.ListMap;
import org.hisp.dhis.common.NameableObject;
import org.hisp.dhis.period.Period;
@@ -60,21 +63,46 @@
return periods;
}
-
- public static String getTableName( Period period, String tableName )
- {
- Period year = PERIODTYPE.createPeriod( period.getStartDate() );
-
- return tableName + SEP + year.getIsoDate();
- }
-
- public static ListMap<String, NameableObject> getTableNamePeriodMap( List<NameableObject> periods, String tableName )
- {
- ListMap<String, NameableObject> map = new ListMap<String, NameableObject>();
-
- for ( NameableObject period : periods )
- {
- map.putValue( getTableName( (Period) period, tableName ), period );
+
+ //TODO allow periods spanning more than two years
+ //TODO optimize by including required filter periods only
+
+ public static Partitions getPartitions( Period period, String tableName )
+ {
+ Partitions partitions = new Partitions();
+
+ Period startYear = PERIODTYPE.createPeriod( period.getStartDate() );
+ Period endYear = PERIODTYPE.createPeriod( period.getEndDate() );
+
+ partitions.add( tableName + SEP + startYear.getIsoDate() );
+
+ if ( !startYear.equals( endYear ) )
+ {
+ partitions.add( tableName + SEP + endYear.getIsoDate() );
+ }
+
+ return partitions;
+ }
+
+ public static Partitions getPartitions( List<NameableObject> periods, String tableName )
+ {
+ Set<String> partitions = new HashSet<String>();
+
+ for ( NameableObject period : periods )
+ {
+ partitions.addAll( getPartitions( (Period) period, tableName ).getPartitions() );
+ }
+
+ return new Partitions( new ArrayList<String>( partitions ) );
+ }
+
+ public static ListMap<Partitions, NameableObject> getPartitionPeriodMap( List<NameableObject> periods, String tableName )
+ {
+ ListMap<Partitions, NameableObject> map = new ListMap<Partitions, NameableObject>();
+
+ for ( NameableObject period : periods )
+ {
+ map.putValue( getPartitions( (Period) period, tableName ), period );
}
return map;
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java 2013-08-23 16:05:01 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java 2013-08-31 18:07:17 +0000
@@ -583,9 +583,7 @@
{
assertDimensionNameNotNull( query );
- assertTrue( query.filterSpansMultiplePartitions() );
- assertEquals( 2, query.getTableNamePeriodMap().size() );
- assertEquals( 2, query.getPartitionFilterParams().size() );
+ assertTrue( query.spansMultiplePartitions() );
}
}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/PartitionUtilsTest.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/PartitionUtilsTest.java 2013-08-23 16:05:01 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/PartitionUtilsTest.java 2013-08-31 18:07:17 +0000
@@ -34,6 +34,10 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.hisp.dhis.analytics.Partitions;
import org.hisp.dhis.common.ListMap;
import org.hisp.dhis.common.NameableObject;
import org.junit.Test;
@@ -43,31 +47,59 @@
*/
public class PartitionUtilsTest
{
- private static final String TABLE_NAME = ANALYTICS_TABLE_NAME;
-
- @Test
- public void testGetTable()
- {
- assertEquals( TABLE_NAME + "_2000", PartitionUtils.getTableName( createPeriod( "200011" ), TABLE_NAME ) );
- assertEquals( TABLE_NAME + "_2001", PartitionUtils.getTableName( createPeriod( "2001W02" ), TABLE_NAME ) );
- assertEquals( TABLE_NAME + "_2002", PartitionUtils.getTableName( createPeriod( "2002Q2" ), TABLE_NAME ) );
- assertEquals( TABLE_NAME + "_2003", PartitionUtils.getTableName( createPeriod( "2003S2" ), TABLE_NAME ) );
- }
-
- @Test
- public void testGetTablePeriodMap()
+ private static final String TBL = ANALYTICS_TABLE_NAME;
+
+ @Test
+ public void testGetPartitions()
+ {
+ assertEquals( new Partitions().add( TBL + "_2000" ), PartitionUtils.getPartitions( createPeriod( "200001" ), TBL ) );
+ assertEquals( new Partitions().add( TBL + "_2001" ), PartitionUtils.getPartitions( createPeriod( "200110" ), TBL ) );
+ assertEquals( new Partitions().add( TBL + "_2002" ), PartitionUtils.getPartitions( createPeriod( "2002Q2" ), TBL ) );
+ assertEquals( new Partitions().add( TBL + "_2003" ), PartitionUtils.getPartitions( createPeriod( "2003S2" ), TBL ) );
+
+ assertEquals( new Partitions().add( TBL + "_2000" ).add( TBL + "_2001" ), PartitionUtils.getPartitions( createPeriod( "2000July" ), TBL ) );
+ assertEquals( new Partitions().add( TBL + "_2001" ).add( TBL + "_2002" ), PartitionUtils.getPartitions( createPeriod( "2001April" ), TBL ) );
+ }
+
+ public void getGetPartitionsMultiplePeriods()
+ {
+ List<NameableObject> periods = new ArrayList<NameableObject>();
+ periods.add( createPeriod( "200011" ) );
+ periods.add( createPeriod( "200105" ) );
+ periods.add( createPeriod( "200108" ) );
+
+ assertEquals( new Partitions().add( TBL + "_2000" ).add( TBL + "_2001" ), PartitionUtils.getPartitions( periods, TBL ) );
+ }
+
+ @Test
+ public void testGetTablePeriodMapA()
{
- ListMap<String, NameableObject> map = PartitionUtils.getTableNamePeriodMap( getList(
- createPeriod( "2000S1" ), createPeriod( "2000S2" ), createPeriod( "2001S1" ), createPeriod( "2001S2" ), createPeriod( "2002S1" ) ), TABLE_NAME );
+ ListMap<Partitions, NameableObject> map = PartitionUtils.getPartitionPeriodMap( getList(
+ createPeriod( "2000S1" ), createPeriod( "2000S2" ), createPeriod( "2001S1" ), createPeriod( "2001S2" ), createPeriod( "2002S1" ) ), TBL );
assertEquals( 3, map.size() );
- assertTrue( map.keySet().contains( TABLE_NAME + "_2000" ) );
- assertTrue( map.keySet().contains( TABLE_NAME + "_2001" ) );
- assertTrue( map.keySet().contains( TABLE_NAME + "_2002" ) );
-
- assertEquals( 2, map.get( TABLE_NAME + "_2000" ).size() );
- assertEquals( 2, map.get( TABLE_NAME + "_2001" ).size() );
- assertEquals( 1, map.get( TABLE_NAME + "_2002" ).size() );
+ assertTrue( map.keySet().contains( new Partitions().add( TBL + "_2000" ) ) );
+ assertTrue( map.keySet().contains( new Partitions().add( TBL + "_2001" ) ) );
+ assertTrue( map.keySet().contains( new Partitions().add( TBL + "_2002" ) ) );
+
+ assertEquals( 2, map.get( new Partitions().add( TBL + "_2000" ) ).size() );
+ assertEquals( 2, map.get( new Partitions().add( TBL + "_2001" ) ).size() );
+ assertEquals( 1, map.get( new Partitions().add( TBL + "_2002" ) ).size() );
+ }
+
+ @Test
+ public void testGetTablePeriodMapB()
+ {
+ ListMap<Partitions, NameableObject> map = PartitionUtils.getPartitionPeriodMap( getList(
+ createPeriod( "2000April" ), createPeriod( "2000" ), createPeriod( "2001" ), createPeriod( "2001Oct" ), createPeriod( "2002Oct" ) ), TBL );
+
+ assertEquals( 5, map.size() );
+
+ assertTrue( map.keySet().contains( new Partitions().add( TBL + "_2000" ) ) );
+ assertTrue( map.keySet().contains( new Partitions().add( TBL + "_2001" ) ) );
+ assertTrue( map.keySet().contains( new Partitions().add( TBL + "_2000" ).add( TBL + "_2001" ) ) );
+ assertTrue( map.keySet().contains( new Partitions().add( TBL + "_2001" ).add( TBL + "_2002" ) ) );
+ assertTrue( map.keySet().contains( new Partitions().add( TBL + "_2002" ).add( TBL + "_2003" ) ) );
}
}
