dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #20378
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 9271: Analytics, impl query planner
------------------------------------------------------------
revno: 9271
committer: Lars Helge Øverland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Tue 2012-12-11 21:46:36 +0100
message:
Analytics, impl query planner
added:
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/QueryPlanner.java
dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/ListMap.java
renamed:
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/ShardUtils.java => dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/PartitionUtils.java
dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/ShardUtilsTest.java => dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/PartitionUtilsTest.java
modified:
dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitService.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsManager.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsService.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java
dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitService.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/PartitionUtils.java
dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/PartitionUtilsTest.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitService.java'
--- dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitService.java 2012-12-11 09:10:15 +0000
+++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitService.java 2012-12-11 20:46:36 +0000
@@ -199,6 +199,13 @@
int getLevelOfOrganisationUnit( int id );
/**
+ * Returns the level of the organisation unit with the given uid.
+ *
+ * @return the level of the organisation unit with the given uid.
+ */
+ int getLevelOfOrganisationUnit( String uid );
+
+ /**
* Returns all OrganisationUnits which are part of the subtree of the
* OrganisationUnit with the given identifer and have no children.
*
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsManager.java 2012-12-03 21:04:39 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsManager.java 2012-12-11 20:46:36 +0000
@@ -27,7 +27,6 @@
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
-import java.util.Collection;
import java.util.List;
import java.util.concurrent.Future;
@@ -35,6 +34,5 @@
public interface AnalyticsManager
{
- Future<List<AggregatedDataValue>> getAggregatedDataValueTotals( Collection<AggregatedDataValue> values, Collection<Integer> dataElementIds,
- Collection<String> periodIds, Collection<Integer> organisationUnitIds );
+ Future<List<AggregatedDataValue>> getAggregatedDataValueTotals( DataQueryParams params );
}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsService.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsService.java 2012-12-03 21:04:39 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsService.java 2012-12-11 20:46:36 +0000
@@ -27,13 +27,11 @@
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
-import java.util.Collection;
import java.util.List;
import org.hisp.dhis.aggregation.AggregatedDataValue;
public interface AnalyticsService
{
- List<AggregatedDataValue> getAggregatedDataValueTotals( Collection<Integer> dataElementIds,
- Collection<String> periodIds, Collection<Integer> organisationUnitIds ) throws Exception;
+ List<AggregatedDataValue> getAggregatedDataValueTotals( DataQueryParams params ) throws Exception;
}
=== added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java 1970-01-01 00:00:00 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java 2012-12-11 20:46:36 +0000
@@ -0,0 +1,255 @@
+package org.hisp.dhis.analytics;
+
+/*
+ * Copyright (c) 2004-2012, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.hisp.dhis.common.Dxf2Namespace;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
+
+@JacksonXmlRootElement( localName = "dxf2", namespace = Dxf2Namespace.NAMESPACE )
+public class DataQueryParams
+{
+ public static final String INDICATOR_DIM_ID = "in";
+ public static final String DATAELEMENT_DIM_ID = "de";
+ public static final String PERIOD_DIM_ID = "pe";
+ public static final String ORGUNIT_DIM_ID = "ou";
+
+ private List<String> indicators = new ArrayList<String>();
+
+ private List<String> dataElements = new ArrayList<String>();
+
+ private List<String> periods = new ArrayList<String>();
+
+ private List<String> organisationUnits = new ArrayList<String>();
+
+ private Map<String, List<String>> dimensions = new HashMap<String, List<String>>();
+
+ private boolean categories = false;
+
+ private transient String tableName;
+
+ // -------------------------------------------------------------------------
+ // Constructors
+ // -------------------------------------------------------------------------
+
+ public DataQueryParams()
+ {
+ }
+
+ public DataQueryParams( List<String> indicators, List<String> dataElements, List<String> periods,
+ List<String> organisationUnits, Map<String, List<String>> dimensions, boolean categories )
+ {
+ this.indicators = indicators;
+ this.dataElements = dataElements;
+ this.periods = periods;
+ this.organisationUnits = organisationUnits;
+ this.dimensions = dimensions;
+ this.categories = categories;
+ }
+
+ public DataQueryParams( DataQueryParams params )
+ {
+ this.indicators = new ArrayList<String>( params.getIndicators() );
+ this.dataElements = new ArrayList<String>( params.getDataElements() );
+ this.periods = new ArrayList<String>( params.getPeriods() );
+ this.organisationUnits = new ArrayList<String>( params.getOrganisationUnits() );
+ this.dimensions = new HashMap<String, List<String>>( params.getDimensions() );
+ this.categories = params.isCategories();
+ }
+
+ // -------------------------------------------------------------------------
+ // Logic
+ // -------------------------------------------------------------------------
+
+ public SortedMap<String, List<String>> getDimensionValuesMap()
+ {
+ SortedMap<String, List<String>> map = new TreeMap<String, List<String>>();
+
+ // TODO convert indicators to data elements
+
+ map.put( DATAELEMENT_DIM_ID, new ArrayList<String>( dataElements ) );
+ map.put( ORGUNIT_DIM_ID, new ArrayList<String>( organisationUnits ) );
+ map.put( PERIOD_DIM_ID, new ArrayList<String>( periods ) );
+
+ for ( String dimension : dimensions.keySet() )
+ {
+ map.put( dimension, dimensions.get( dimension ) );
+ }
+
+ return map;
+ }
+
+ public void setDimension( String dimension, List<String> values )
+ {
+ if ( DATAELEMENT_DIM_ID.equals( dimension ) )
+ {
+ setDataElements( values );
+ }
+ else if ( PERIOD_DIM_ID.equals( dimension ) )
+ {
+ setPeriods( values );
+ }
+ else if ( ORGUNIT_DIM_ID.equals( dimension ) )
+ {
+ setOrganisationUnits( values );
+ }
+ else if ( dimensions.containsKey( dimension ) )
+ {
+ dimensions.put( dimension, values );
+ }
+ }
+
+ public List<String> getDimension( String dimension )
+ {
+ if ( DATAELEMENT_DIM_ID.equals( dimension ) )
+ {
+ return dataElements;
+ }
+ else if ( PERIOD_DIM_ID.equals( dimension ) )
+ {
+ return periods;
+ }
+ else if ( ORGUNIT_DIM_ID.equals( dimension ) )
+ {
+ return organisationUnits;
+ }
+ else if ( dimensions.containsKey( dimension ) )
+ {
+ return dimensions.get( dimension );
+ }
+
+ throw new IllegalArgumentException( dimension );
+ }
+
+ public String getLargestDimension()
+ {
+ Map<String, List<String>> map = getDimensionValuesMap();
+
+ String dimension = map.keySet().iterator().next();
+ int size = map.get( dimension ).size();
+
+ for ( String dim : map.keySet() )
+ {
+ if ( map.get( dim ).size() > size )
+ {
+ dimension = dim;
+ size = map.get( dim ).size();
+ }
+ }
+
+ return dimension;
+ }
+
+ // -------------------------------------------------------------------------
+ // Get and set methods
+ // -------------------------------------------------------------------------
+
+ @JsonProperty( value = INDICATOR_DIM_ID )
+ public List<String> getIndicators()
+ {
+ return indicators;
+ }
+
+ public void setIndicators( List<String> indicators )
+ {
+ this.indicators = indicators;
+ }
+
+ @JsonProperty( value = DATAELEMENT_DIM_ID )
+ public List<String> getDataElements()
+ {
+ return dataElements;
+ }
+
+ public void setDataElements( List<String> dataElements )
+ {
+ this.dataElements = dataElements;
+ }
+
+ @JsonProperty( value = PERIOD_DIM_ID )
+ public List<String> getPeriods()
+ {
+ return periods;
+ }
+
+ public void setPeriods( List<String> periods )
+ {
+ this.periods = periods;
+ }
+
+ @JsonProperty( value = ORGUNIT_DIM_ID )
+ public List<String> getOrganisationUnits()
+ {
+ return organisationUnits;
+ }
+
+ public void setOrganisationUnits( List<String> organisationUnits )
+ {
+ this.organisationUnits = organisationUnits;
+ }
+
+ @JsonProperty( value = "dimensions" )
+ public Map<String, List<String>> getDimensions()
+ {
+ return dimensions;
+ }
+
+ public void setDimensions( Map<String, List<String>> dimensions )
+ {
+ this.dimensions = dimensions;
+ }
+
+ @JsonProperty( value = "categories" )
+ public boolean isCategories()
+ {
+ return categories;
+ }
+
+ public void setCategories( boolean categories )
+ {
+ this.categories = categories;
+ }
+
+ public String getTableName()
+ {
+ return tableName;
+ }
+
+ public void setTableName( String tableName )
+ {
+ this.tableName = tableName;
+ }
+}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java 2012-12-03 21:04:39 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java 2012-12-11 20:46:36 +0000
@@ -28,37 +28,40 @@
*/
import java.util.ArrayList;
-import java.util.Collection;
import java.util.List;
import java.util.concurrent.Future;
import org.hisp.dhis.aggregation.AggregatedDataValue;
import org.hisp.dhis.analytics.AnalyticsManager;
import org.hisp.dhis.analytics.AnalyticsService;
-import org.hisp.dhis.system.util.PaginatedList;
+import org.hisp.dhis.analytics.DataQueryParams;
import org.hisp.dhis.system.util.Timer;
import org.springframework.beans.factory.annotation.Autowired;
public class DefaultAnalyticsService
implements AnalyticsService
{
+ //TODO select from correct shard
+ //TODO period aggregation for multiple period types
+ //TODO hierarchy aggregation for org units at multiple levels
+ //TODO indicator aggregation
+
@Autowired
private AnalyticsManager analyticsManager;
- public List<AggregatedDataValue> getAggregatedDataValueTotals( Collection<Integer> dataElementIds,
- Collection<String> periodIds, Collection<Integer> organisationUnitIds ) throws Exception
+ public List<AggregatedDataValue> getAggregatedDataValueTotals( DataQueryParams params ) throws Exception
{
Timer t = new Timer().start();
- List<List<Integer>> dePages = new PaginatedList<Integer>( dataElementIds ).setNumberOfPages( 4 ).getPages();
+ List<DataQueryParams> queries = QueryPlanner.planQuery( params, 6 );
List<Future<List<AggregatedDataValue>>> futures = new ArrayList<Future<List<AggregatedDataValue>>>();
List<AggregatedDataValue> values = new ArrayList<AggregatedDataValue>();
- for ( List<Integer> dePage : dePages )
+ for ( DataQueryParams query : queries )
{
- futures.add( analyticsManager.getAggregatedDataValueTotals( values, dePage, periodIds, organisationUnitIds ) );
+ futures.add( analyticsManager.getAggregatedDataValueTotals( query ) );
}
for ( Future<List<AggregatedDataValue>> future : futures )
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java 2012-12-03 21:04:39 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java 2012-12-11 20:46:36 +0000
@@ -27,10 +27,8 @@
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
-import static org.hisp.dhis.system.util.TextUtils.getCommaDelimitedString;
import static org.hisp.dhis.system.util.TextUtils.getQuotedCommaDelimitedString;
-import java.util.Collection;
import java.util.List;
import java.util.concurrent.Future;
@@ -38,6 +36,7 @@
import org.apache.commons.logging.LogFactory;
import org.hisp.dhis.aggregation.AggregatedDataValue;
import org.hisp.dhis.analytics.AnalyticsManager;
+import org.hisp.dhis.analytics.DataQueryParams;
import org.hisp.dhis.expression.ExpressionService;
import org.hisp.dhis.organisationunit.OrganisationUnitService;
import org.hisp.dhis.period.PeriodService;
@@ -76,23 +75,18 @@
// Implementation
// -------------------------------------------------------------------------
- //TODO period aggregation for multiple period types
- //TODO hierarchy aggregation for org units at multiple levels
- //TODO indicator aggregation
-
@Async
- public Future<List<AggregatedDataValue>> getAggregatedDataValueTotals( Collection<AggregatedDataValue> values, Collection<Integer> dataElementIds,
- Collection<String> periodIds, Collection<Integer> organisationUnitIds )
+ public Future<List<AggregatedDataValue>> getAggregatedDataValueTotals( DataQueryParams params )
{
- int level = organisationUnitService.getLevelOfOrganisationUnit( organisationUnitIds.iterator().next() );
- String periodType = PeriodType.getPeriodTypeFromIsoString( periodIds.iterator().next() ).getName().toLowerCase();
+ int level = organisationUnitService.getLevelOfOrganisationUnit( params.getOrganisationUnits().iterator().next() );
+ String periodType = PeriodType.getPeriodTypeFromIsoString( params.getPeriods().iterator().next() ).getName().toLowerCase();
final String sql =
"SELECT dataelementid, 0 as categoryoptioncomboid, periodid, idlevel" + level + " as organisationunitid, SUM(value) as value " +
- "FROM analytics " +
- "WHERE dataelementid IN ( " + getCommaDelimitedString( dataElementIds ) + " ) " +
- "AND " + periodType + " IN ( " + getQuotedCommaDelimitedString( periodIds ) + " ) " +
- "AND idlevel" + level + " IN ( " + getCommaDelimitedString( organisationUnitIds ) + " ) " +
+ "FROM " + params.getTableName() + " " +
+ "WHERE dataelementid IN ( " + getQuotedCommaDelimitedString( params.getDataElements() ) + " ) " +
+ "AND " + periodType + " IN ( " + getQuotedCommaDelimitedString( params.getPeriods() ) + " ) " +
+ "AND idlevel" + level + " IN ( " + getQuotedCommaDelimitedString( params.getOrganisationUnits() ) + " ) " +
"GROUP BY dataelementid, periodid, idlevel" + level;
log.info( sql );
=== added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/QueryPlanner.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/QueryPlanner.java 1970-01-01 00:00:00 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/QueryPlanner.java 2012-12-11 20:46:36 +0000
@@ -0,0 +1,106 @@
+package org.hisp.dhis.analytics.data;
+
+/*
+ * Copyright (c) 2004-2012, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.SortedMap;
+
+import org.hisp.dhis.analytics.DataQueryParams;
+import org.hisp.dhis.analytics.table.PartitionUtils;
+import org.hisp.dhis.system.util.ListMap;
+import org.hisp.dhis.system.util.PaginatedList;
+
+public class QueryPlanner
+{
+ /**
+ * Creates a list of DataParams.
+ *
+ * @param params the data query params.
+ * @param optimalQueries the number of optimal queries for the planner to return.
+ * @return
+ */
+ public static List<DataQueryParams> planQuery( DataQueryParams params, int optimalQueries )
+ {
+ List<DataQueryParams> queries = new ArrayList<DataQueryParams>();
+
+ ListMap<String, String> tablePeriodMap = PartitionUtils.getTablePeriodMap( params.getPeriods() );
+
+ boolean periodSatisfies = tablePeriodMap.size() >= optimalQueries;
+
+ if ( periodSatisfies )
+ {
+ for ( String tableName : tablePeriodMap.keySet() )
+ {
+ DataQueryParams query = new DataQueryParams( params );
+ query.setPeriods( tablePeriodMap.get( tableName ) );
+ query.setTableName( tableName );
+ queries.add( query );
+ }
+ }
+ else
+ {
+ int pages = optimalQueries / tablePeriodMap.size();
+
+ String dimension = getPartitionDimension( params, pages );
+
+ List<String> dimensionValues = params.getDimension( dimension );
+
+ List<List<String>> valuePages = new PaginatedList<String>( dimensionValues ).setNumberOfPages( pages ).getPages();
+
+ for ( String tableName : tablePeriodMap.keySet() )
+ {
+ for ( List<String> values : valuePages )
+ {
+ DataQueryParams query = new DataQueryParams( params );
+ query.setPeriods( tablePeriodMap.get( tableName ) );
+ query.setDimension( dimension, values );
+ query.setTableName( tableName );
+ queries.add( query );
+ }
+ }
+ }
+
+ return queries;
+ }
+
+ private static String getPartitionDimension( DataQueryParams params, int optimalQueries )
+ {
+ SortedMap<String, List<String>> map = params.getDimensionValuesMap();
+
+ for ( String dimension : map.keySet() )
+ {
+ if ( map.get( dimension ).size() >= optimalQueries )
+ {
+ return dimension;
+ }
+ }
+
+ return params.getLargestDimension();
+ }
+}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java 2012-12-11 01:38:43 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java 2012-12-11 20:46:36 +0000
@@ -67,7 +67,7 @@
final Date earliest = tableManager.getEarliestData();
final Date latest = tableManager.getLatestData();
- final List<String> tables = ShardUtils.getTempTableNames( earliest, latest );
+ final List<String> tables = PartitionUtils.getTempTableNames( earliest, latest );
clock.logTime( "Checked data timespan" );
//dropTables( tables ); //remove
@@ -116,7 +116,7 @@
for ( String table : tablePage )
{
- Period period = ShardUtils.getPeriod( table );
+ Period period = PartitionUtils.getPeriod( table );
futures.add( tableManager.populateTableAsync( table, period.getStartDate(), period.getEndDate() ) );
}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java 2012-12-11 01:38:43 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java 2012-12-11 20:46:36 +0000
@@ -54,9 +54,13 @@
* each organisation unit group set and organisation unit level. Also, columns
* for dataelementid, periodid, organisationunitid, categoryoptioncomboid, value.
*
+ * The analytics table is horizontally partitioned. The partition key is the start
+ * date of the period of the data record. The table is partitioned according to
+ * time span with one partition per calendar quarter.
+ *
* The data records in this table are not aggregated. Typically, queries will
* aggregate in organisation unit hierarchy dimension, in the period/time dimension,
- * and the category dimensions, as well as org unit group set dimensions.
+ * and the category dimensions, as well as organisation unit group set dimensions.
*
* @author Lars Helge Overland
*/
@@ -82,7 +86,6 @@
// Implementation
// -------------------------------------------------------------------------
- //TODO shard on data quarter based on start date
//TODO average aggregation operator data, pre-aggregate in time dimension, not in org unit dimension
public void createTable( String tableName )
=== renamed file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/ShardUtils.java' => 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/PartitionUtils.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/ShardUtils.java 2012-12-11 01:38:43 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/PartitionUtils.java 2012-12-11 20:46:36 +0000
@@ -31,14 +31,16 @@
import static org.hisp.dhis.analytics.AnalyticsTableManager.TABLE_NAME_TEMP;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Date;
import java.util.List;
import org.hisp.dhis.period.Period;
import org.hisp.dhis.period.PeriodType;
import org.hisp.dhis.period.QuarterlyPeriodType;
+import org.hisp.dhis.system.util.ListMap;
-public class ShardUtils
+public class PartitionUtils
{
private static final QuarterlyPeriodType QUARTERLY = new QuarterlyPeriodType();
@@ -93,4 +95,16 @@
return PeriodType.getPeriodFromIsoString( isoPeriod );
}
+
+ public static ListMap<String, String> getTablePeriodMap( Collection<String> isoPeriods )
+ {
+ ListMap<String, String> map = new ListMap<String, String>();
+
+ for ( String period : isoPeriods )
+ {
+ map.putValue( getTable( period ), period );
+ }
+
+ return map;
+ }
}
=== renamed file 'dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/ShardUtilsTest.java' => 'dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/PartitionUtilsTest.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/ShardUtilsTest.java 2012-12-11 01:38:43 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/PartitionUtilsTest.java 2012-12-11 20:46:36 +0000
@@ -40,7 +40,7 @@
import org.hisp.dhis.period.QuarterlyPeriodType;
import org.junit.Test;
-public class ShardUtilsTest
+public class PartitionUtilsTest
{
@Test
public void testGetTableNames()
@@ -49,7 +49,7 @@
Date earliest = cal.set( 2000, 5, 4 ).time();
Date latest = cal.set( 2001, 2, 10 ).time();
- List<String> tables = ShardUtils.getTempTableNames( earliest, latest );
+ List<String> tables = PartitionUtils.getTempTableNames( earliest, latest );
assertEquals( 4, tables.size() );
assertTrue( tables.contains( TABLE_NAME_TEMP + "_2000Q2" ) );
@@ -61,11 +61,11 @@
@Test
public void testGetTable()
{
- assertEquals( TABLE_NAME + "_2000Q4", ShardUtils.getTable( "200011" ) );
- assertEquals( TABLE_NAME + "_2000Q1", ShardUtils.getTable( "2000W02" ) );
- assertEquals( TABLE_NAME + "_2000Q2", ShardUtils.getTable( "2000Q2" ) );
- assertEquals( TABLE_NAME + "_2000Q3", ShardUtils.getTable( "2000S2" ) );
- assertEquals( TABLE_NAME + "_2000Q1", ShardUtils.getTable( "2000" ) );
+ assertEquals( TABLE_NAME + "_2000Q4", PartitionUtils.getTable( "200011" ) );
+ assertEquals( TABLE_NAME + "_2000Q1", PartitionUtils.getTable( "2000W02" ) );
+ assertEquals( TABLE_NAME + "_2000Q2", PartitionUtils.getTable( "2000Q2" ) );
+ assertEquals( TABLE_NAME + "_2000Q3", PartitionUtils.getTable( "2000S2" ) );
+ assertEquals( TABLE_NAME + "_2000Q1", PartitionUtils.getTable( "2000" ) );
}
@Test
@@ -76,7 +76,7 @@
Period q2 = new QuarterlyPeriodType().createPeriod( cal.set( 2000, 4, 1 ).time() );
Period q4 = new QuarterlyPeriodType().createPeriod( cal.set( 2000, 10, 1 ).time() );
- assertEquals( q2, ShardUtils.getPeriod( TABLE_NAME_TEMP + "_2000Q2" ) );
- assertEquals( q4, ShardUtils.getPeriod( TABLE_NAME_TEMP + "_2000Q4" ) );
+ assertEquals( q2, PartitionUtils.getPeriod( TABLE_NAME_TEMP + "_2000Q2" ) );
+ assertEquals( q4, PartitionUtils.getPeriod( TABLE_NAME_TEMP + "_2000Q4" ) );
}
}
=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitService.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitService.java 2012-12-11 09:10:15 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitService.java 2012-12-11 20:46:36 +0000
@@ -230,6 +230,11 @@
return getOrganisationUnit( id ).getOrganisationUnitLevel();
}
+ public int getLevelOfOrganisationUnit( String uid )
+ {
+ return getOrganisationUnit( uid ).getOrganisationUnitLevel();
+ }
+
public Collection<OrganisationUnit> getLeafOrganisationUnits( int id )
{
Collection<OrganisationUnit> units = getOrganisationUnitWithChildren( id );
=== added file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/ListMap.java'
--- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/ListMap.java 1970-01-01 00:00:00 +0000
+++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/ListMap.java 2012-12-11 20:46:36 +0000
@@ -0,0 +1,18 @@
+package org.hisp.dhis.system.util;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+public class ListMap<T, V>
+ extends HashMap<T, List<V>>
+{
+ public List<String> putValue( T key, V value )
+ {
+ List<V> list = this.get( key );
+ list = list == null ? new ArrayList<V>() : list;
+ list.add( value );
+ this.put( key, list );
+ return null;
+ }
+}
