dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #33548
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 17139: Analytics. Added validation of analytics tables.
------------------------------------------------------------
revno: 17139
committer: Lars Helge Overland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Thu 2014-10-16 16:51:51 +0200
message:
Analytics. Added validation of analytics tables.
modified:
dhis-2/dhis-api/src/main/java/org/hisp/dhis/util/Timer.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsService.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java
dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/ListUtils.java
dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/util/ListUtilsTest.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/util/Timer.java'
--- dhis-2/dhis-api/src/main/java/org/hisp/dhis/util/Timer.java 2014-10-07 12:16:23 +0000
+++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/util/Timer.java 2014-10-16 14:51:51 +0000
@@ -38,7 +38,7 @@
private long startTime;
private boolean printDisabled;
-
+
public Timer disablePrint()
{
this.printDisabled = true;
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java 2014-10-16 06:17:19 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java 2014-10-16 14:51:51 +0000
@@ -577,8 +577,6 @@
Grid grid = getAggregatedDataValues( params );
Map<String, Double> valueMap = getAggregatedDataValueMapping( grid );
-
- log.info( "Got aggregated values for table layout" );
return reportTable.getGrid( new ListGrid( grid.getMetaData() ), valueMap, false );
}
@@ -743,11 +741,11 @@
int optimalQueries = MathUtils.getWithin( getProcessNo(), 1, MAX_QUERIES );
- Timer t = new Timer().start();
+ Timer t = new Timer().start().disablePrint();
DataQueryGroups queryGroups = queryPlanner.planQuery( params, optimalQueries, tableName );
- log.debug( "Planned analytics query, got: " + queryGroups.getLargestGroupSize() + " for optimal: " + optimalQueries );
+ t.getSplitTime( "Planned analytics query, got: " + queryGroups.getLargestGroupSize() + " for optimal: " + optimalQueries );
Map<String, Object> map = new HashMap<>();
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsService.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsService.java 2014-10-16 12:28:42 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsService.java 2014-10-16 14:51:51 +0000
@@ -48,8 +48,6 @@
import java.util.Map;
import java.util.Set;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.hisp.dhis.analytics.AnalyticsSecurityManager;
import org.hisp.dhis.analytics.AnalyticsService;
import org.hisp.dhis.analytics.SortOrder;
@@ -85,9 +83,9 @@
import org.hisp.dhis.system.grid.ListGrid;
import org.hisp.dhis.system.util.DateUtils;
import org.hisp.dhis.system.util.ListUtils;
-import org.hisp.dhis.util.Timer;
import org.hisp.dhis.trackedentity.TrackedEntityAttribute;
import org.hisp.dhis.trackedentity.TrackedEntityAttributeService;
+import org.hisp.dhis.util.Timer;
import org.springframework.beans.factory.annotation.Autowired;
/**
@@ -95,9 +93,7 @@
*/
public class DefaultEventAnalyticsService
implements EventAnalyticsService
-{
- private static final Log log = LogFactory.getLog( DefaultEventAnalyticsService.class );
-
+{
private static final String ITEM_EVENT = "psi";
private static final String ITEM_PROGRAM_STAGE = "ps";
private static final String ITEM_EXECUTION_DATE = "eventdate";
@@ -178,11 +174,11 @@
// Data
// ---------------------------------------------------------------------
- Timer t = new Timer().start();
+ Timer t = new Timer().start().disablePrint();
List<EventQueryParams> queries = queryPlanner.planAggregateQuery( params );
- log.debug( "Planned event query, got partitions: " + params.getPartitions() );
+ t.getSplitTime( "Planned event query, got partitions: " + params.getPartitions() );
for ( EventQueryParams query : queries )
{
@@ -285,7 +281,7 @@
params = queryPlanner.planEventQuery( params );
- log.debug( "Planned event query, got partitions: " + params.getPartitions() );
+ t.getSplitTime( "Planned event query, got partitions: " + params.getPartitions() );
int count = 0;
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java 2014-10-16 06:17:19 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java 2014-10-16 14:51:51 +0000
@@ -32,6 +32,7 @@
import java.util.Calendar;
import java.util.Date;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.Future;
@@ -50,6 +51,7 @@
import org.hisp.dhis.period.Cal;
import org.hisp.dhis.period.Period;
import org.hisp.dhis.setting.SystemSettingManager;
+import org.hisp.dhis.system.util.ListUtils;
import org.joda.time.DateTime;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.BadSqlGrammarException;
@@ -328,4 +330,30 @@
log.debug( ex.getMessage() );
}
}
+
+ /**
+ * Checks whether the given list of dimensions are valid.
+ * @throws IllegalStateException if not valid.
+ */
+ protected void validateDimensionColumns( List<String[]> dimensions )
+ {
+ if ( dimensions == null || dimensions.isEmpty() )
+ {
+ throw new IllegalStateException( "Analytics table dimensions are empty" );
+ }
+
+ List<String> columns = new ArrayList<>();
+
+ for ( String[] dimension : dimensions )
+ {
+ columns.add( dimension[0] );
+ }
+
+ Set<String> duplicates = ListUtils.getDuplicates( columns );
+
+ if ( !duplicates.isEmpty() )
+ {
+ throw new IllegalStateException( "Analytics table dimensions contain duplicates: " + duplicates );
+ }
+ }
}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java 2014-10-16 06:17:19 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java 2014-10-16 14:51:51 +0000
@@ -114,8 +114,12 @@
executeSilently( sqlDrop );
String sqlCreate = "create table " + tableName + " (";
-
- for ( String[] col : getDimensionColumns( table ) )
+
+ List<String[]> columns = getDimensionColumns( table );
+
+ validateDimensionColumns( columns );
+
+ for ( String[] col : columns )
{
sqlCreate += col[0] + " " + col[1] + ",";
}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java 2014-10-16 06:17:19 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java 2014-10-16 14:51:51 +0000
@@ -77,8 +77,12 @@
executeSilently( sqlDrop );
String sqlCreate = "create table " + tableName + " (";
-
- for ( String[] col : getDimensionColumns( table ) )
+
+ List<String[]> columns = getDimensionColumns( table );
+
+ validateDimensionColumns( columns );
+
+ for ( String[] col : columns )
{
sqlCreate += col[0] + " " + col[1] + ",";
}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java 2014-10-16 06:17:19 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java 2014-10-16 14:51:51 +0000
@@ -79,7 +79,11 @@
String sqlCreate = "create table " + tableName + " (";
- for ( String[] col : getDimensionColumns( table ) )
+ List<String[]> columns = getDimensionColumns( table );
+
+ validateDimensionColumns( columns );
+
+ for ( String[] col : columns )
{
sqlCreate += col[0] + " " + col[1] + ",";
}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java 2014-10-16 06:17:19 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java 2014-10-16 14:51:51 +0000
@@ -125,7 +125,11 @@
String sqlCreate = "create table " + tableName + " (";
- for ( String[] col : getDimensionColumns( table ) )
+ List<String[]> columns = getDimensionColumns( table );
+
+ validateDimensionColumns( columns );
+
+ for ( String[] col : columns )
{
sqlCreate += col[0] + " " + col[1] + ",";
}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java 2014-10-16 06:17:19 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java 2014-10-16 14:51:51 +0000
@@ -78,7 +78,11 @@
String sqlCreate = "create table " + tableName + " (";
- for ( String[] col : getDimensionColumns( table ) )
+ List<String[]> columns = getDimensionColumns( table );
+
+ validateDimensionColumns( columns );
+
+ for ( String[] col : columns )
{
sqlCreate += col[0] + " " + col[1] + ",";
}
=== modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/ListUtils.java'
--- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/ListUtils.java 2014-09-28 17:12:48 +0000
+++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/ListUtils.java 2014-10-16 14:51:51 +0000
@@ -150,7 +150,7 @@
* @param comparator the comparator.
* @return a set of duplicates from the given list.
*/
- public static <T> Collection<T> getDuplicates( List<T> list, Comparator<T> comparator )
+ public static <T> Set<T> getDuplicates( List<T> list, Comparator<T> comparator )
{
Set<T> duplicates = new HashSet<>();
@@ -170,6 +170,28 @@
return duplicates;
}
+
+ /**
+ * Returns the duplicates in the given list.
+ *
+ * @param list the list.
+ * @return a set of duplicates from the given list.
+ */
+ public static <T> Set<T> getDuplicates( List<T> list )
+ {
+ Set<T> duplicates = new HashSet<>();
+ UniqueArrayList<T> uniqueElements = new UniqueArrayList<>();
+
+ for ( T element : list )
+ {
+ if ( !uniqueElements.add( element ) )
+ {
+ duplicates.add( element );
+ }
+ }
+
+ return duplicates;
+ }
/**
* Returns a Collection with the given items.
=== modified file 'dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/util/ListUtilsTest.java'
--- dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/util/ListUtilsTest.java 2014-08-15 07:40:20 +0000
+++ dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/util/ListUtilsTest.java 2014-10-16 14:51:51 +0000
@@ -30,7 +30,9 @@
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import org.junit.Test;
@@ -58,4 +60,15 @@
assertTrue( list.contains( "e" ) );
assertTrue( list.contains( "g" ) );
}
+
+ @Test
+ public void testGetDuplicates()
+ {
+ List<String> list = new ArrayList<>( Arrays.asList( "a", "b", "c", "c", "d", "e", "e", "e", "f" ) );
+ Set<String> expected = new HashSet<>( Arrays.asList( "c", "e" ) );
+ assertEquals( expected, ListUtils.getDuplicates( list ) );
+
+ list = new ArrayList<>( Arrays.asList( "a", "b", "c", "d", "e", "f", "g", "h" ) );
+ assertEquals( 0, ListUtils.getDuplicates( list ).size() );
+ }
}
