dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #36100
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 18480: Analytics. Better implementation of max records limit, which breaks the query response rather tha...
------------------------------------------------------------
revno: 18480
committer: Lars Helge Overland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Wed 2015-03-04 19:36:19 +0100
message:
Analytics. Better implementation of max records limit, which breaks the query response rather than guessing the possible records.
modified:
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsManager.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java
dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsManager.java 2015-01-17 07:41:26 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsManager.java 2015-03-04 18:36:19 +0000
@@ -31,6 +31,7 @@
import java.util.Map;
import java.util.concurrent.Future;
+import org.hisp.dhis.common.IllegalQueryException;
import org.hisp.dhis.common.ListMap;
import org.hisp.dhis.common.NameableObject;
@@ -46,9 +47,11 @@
* asynchronously. The value class can be Double or String.
*
* @param params the query to retrieve aggregated data for.
+ * @param maxLimit the max number of records to retrieve.
* @return a map.
+ * @throws IllegalQueryException if query result set exceeds the max limit.
*/
- Future<Map<String, Object>> getAggregatedDataValues( DataQueryParams params );
+ Future<Map<String, Object>> getAggregatedDataValues( DataQueryParams params, int maxLimit );
/**
* Inserts entries for the aggregation periods mapped to each data period
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java 2015-02-25 15:59:31 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java 2015-03-04 18:36:19 +0000
@@ -28,8 +28,8 @@
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
+import static org.hisp.dhis.analytics.AggregationType.AVERAGE_INT_DISAGGREGATION;
import static org.hisp.dhis.analytics.AggregationType.AVERAGE_SUM_INT_DISAGGREGATION;
-import static org.hisp.dhis.analytics.AggregationType.AVERAGE_INT_DISAGGREGATION;
import static org.hisp.dhis.common.DimensionType.DATASET;
import static org.hisp.dhis.common.DimensionType.ORGANISATIONUNIT;
import static org.hisp.dhis.common.DimensionType.ORGANISATIONUNIT_GROUPSET;
@@ -44,7 +44,6 @@
import static org.hisp.dhis.common.DimensionalObject.PERIOD_DIM_ID;
import static org.hisp.dhis.common.NameableObjectUtils.asList;
import static org.hisp.dhis.common.NameableObjectUtils.getList;
-import static org.hisp.dhis.system.util.CollectionUtils.emptyIfNull;
import java.util.ArrayList;
import java.util.Arrays;
@@ -65,6 +64,7 @@
import org.hisp.dhis.common.DimensionalObjectUtils;
import org.hisp.dhis.common.DisplayProperty;
import org.hisp.dhis.common.ListMap;
+import org.hisp.dhis.common.MapMap;
import org.hisp.dhis.common.NameableObject;
import org.hisp.dhis.dataelement.DataElement;
import org.hisp.dhis.dataelement.DataElementCategory;
@@ -78,7 +78,6 @@
import org.hisp.dhis.period.PeriodType;
import org.hisp.dhis.system.util.CollectionUtils;
import org.hisp.dhis.system.util.ListUtils;
-import org.hisp.dhis.common.MapMap;
import org.hisp.dhis.system.util.MathUtils;
/**
@@ -544,35 +543,7 @@
return null;
}
-
- /**
- * Returns the number of dimension option permutations. Merges the three data
- * dimensions into one prior to the calculation.
- */
- public int getNumberOfDimensionOptionPermutations()
- {
- int total = 1;
-
- DataQueryParams query = this.instance();
-
- query.getDimensions().add( new BaseDimensionalObject( DATA_X_DIM_ID ) );
-
- query.getDimension( DATA_X_DIM_ID ).getItems().addAll( emptyIfNull( query.getDimensionOptions( INDICATOR_DIM_ID ) ) );
- query.getDimension( DATA_X_DIM_ID ).getItems().addAll( emptyIfNull( query.getDimensionOptions( DATAELEMENT_DIM_ID ) ) );
- query.getDimension( DATA_X_DIM_ID ).getItems().addAll( emptyIfNull( query.getDimensionOptions( DATASET_DIM_ID ) ) );
-
- query.removeDimension( INDICATOR_DIM_ID );
- query.removeDimension( DATAELEMENT_DIM_ID );
- query.removeDimension( DATASET_DIM_ID );
-
- for ( DimensionalObject dim : query.getDimensions() )
- {
- total *= Math.max( dim.getItems().size(), 1 );
- }
-
- return total;
- }
-
+
/**
* Returns a list of dimensions which occur more than once, not including
* the first duplicate.
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java 2015-03-04 17:49:02 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java 2015-03-04 18:36:19 +0000
@@ -737,6 +737,8 @@
int optimalQueries = MathUtils.getWithin( getProcessNo(), 1, MAX_QUERIES );
+ int maxLimit = getMaxLimit();
+
Timer t = new Timer().start().disablePrint();
DataQueryGroups queryGroups = queryPlanner.planQuery( params, optimalQueries, tableName );
@@ -751,7 +753,7 @@
for ( DataQueryParams query : queries )
{
- futures.add( analyticsManager.getAggregatedDataValues( query ) );
+ futures.add( analyticsManager.getAggregatedDataValues( query, maxLimit ) );
}
for ( Future<Map<String, Object>> future : futures )
@@ -1375,4 +1377,12 @@
{
return value != null && Double.class.equals( value.getClass() ) ? MathUtils.getRounded( (Double) value ) : value;
}
+
+ /**
+ * Returns the max records limit. 0 indicates no limit.
+ */
+ private int getMaxLimit()
+ {
+ return (Integer) systemSettingManager.getSystemSetting( SystemSettingManager.KEY_ANALYTICS_MAX_LIMIT, SystemSettingManager.DEFAULT_ANALYTICS_MAX_LIMIT );
+ }
}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java 2015-02-05 18:58:22 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java 2015-03-04 18:36:19 +0000
@@ -140,11 +140,6 @@
violation = "Category option combos cannot be specified as filter";
}
- if ( !params.isIgnoreLimit() && getMaxLimit() > 0 && params.getNumberOfDimensionOptionPermutations() > getMaxLimit() )
- {
- violation = "Table exceeds max number of cells: " + getMaxLimit() + " (" + params.getNumberOfDimensionOptionPermutations() + ")";
- }
-
if ( !params.getDuplicateDimensions().isEmpty() )
{
violation = "Dimensions cannot be specified more than once: " + params.getDuplicateDimensions();
@@ -795,12 +790,4 @@
return map;
}
-
- /**
- * Returns the max records limit. 0 indicates no limit.
- */
- private int getMaxLimit()
- {
- return (Integer) systemSettingManager.getSystemSetting( SystemSettingManager.KEY_ANALYTICS_MAX_LIMIT, SystemSettingManager.DEFAULT_ANALYTICS_MAX_LIMIT );
- }
}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java 2015-01-17 07:41:26 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java 2015-03-04 18:36:19 +0000
@@ -66,6 +66,7 @@
import org.hisp.dhis.analytics.MeasureFilter;
import org.hisp.dhis.common.DimensionalObject;
import org.hisp.dhis.common.DimensionalObjectUtils;
+import org.hisp.dhis.common.IllegalQueryException;
import org.hisp.dhis.common.ListMap;
import org.hisp.dhis.common.NameableObject;
import org.hisp.dhis.jdbc.StatementBuilder;
@@ -112,7 +113,7 @@
@Override
@Async
- public Future<Map<String, Object>> getAggregatedDataValues( DataQueryParams params )
+ public Future<Map<String, Object>> getAggregatedDataValues( DataQueryParams params, int maxLimit )
{
try
{
@@ -139,7 +140,7 @@
try
{
- map = getKeyValueMap( params, sql );
+ map = getKeyValueMap( params, sql, maxLimit );
}
catch ( BadSqlGrammarException ex )
{
@@ -389,8 +390,7 @@
* Retrieves data from the database based on the given query and SQL and puts
* into a value key and value mapping.
*/
- private Map<String, Object> getKeyValueMap( DataQueryParams params, String sql )
- throws BadSqlGrammarException
+ private Map<String, Object> getKeyValueMap( DataQueryParams params, String sql, int maxLimit )
{
Map<String, Object> map = new HashMap<>();
@@ -398,8 +398,15 @@
log.debug( "Analytics SQL: " + sql );
+ int counter = 0;
+
while ( rowSet.next() )
{
+ if ( maxLimit > 0 && ++counter > maxLimit )
+ {
+ throw new IllegalQueryException( "Query result set exceeds max limit: " + maxLimit );
+ }
+
StringBuilder key = new StringBuilder();
for ( DimensionalObject dim : params.getQueryDimensions() )
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java 2015-01-17 07:41:26 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java 2015-03-04 18:36:19 +0000
@@ -337,29 +337,6 @@
assertEquals( PERIOD_DIM_ID, permutation.get( 1 ).getDimension() );
}
}
-
- /**
- * First, combines data elements and data sets into one data dimension and
- * returns (2 + 3) * 3 * 2 = 30. Second, ignores any data dimension and
- * returns 3 * 2 = 6.
- */
- @Test
- public void testGetNumberOfDimensionOptionPermutations()
- {
- DataQueryParams params = new DataQueryParams();
- params.setDataElements( getList( deA, deB ) );
- params.setDataSets( getList( dsA, dsB, dsC ) );
- params.setOrganisationUnits( getList( ouA, ouB, ouC ) );
- params.setPeriods( getList( createPeriod( "2000Q1" ), createPeriod( "2000Q2" ) ) );
-
- assertEquals( 30, params.getNumberOfDimensionOptionPermutations() );
-
- params = new DataQueryParams();
- params.setOrganisationUnits( getList( ouA, ouB, ouC ) );
- params.setPeriods( getList( createPeriod( "2000Q1" ), createPeriod( "2000Q2" ) ) );
-
- assertEquals( 6, params.getNumberOfDimensionOptionPermutations() );
- }
@Test
public void testGetDataPeriodAggregationPeriodMap()
