dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #20516
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 9390: Analytics, implemented support for aggregation levels
------------------------------------------------------------
revno: 9390
committer: Lars Helge Øverland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Sat 2012-12-22 18:51:57 +0100
message:
Analytics, implemented support for aggregation levels
modified:
dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/IdentifiableObjectUtils.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/IdentifiableObjectUtils.java'
--- dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/IdentifiableObjectUtils.java 2012-11-03 19:38:41 +0000
+++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/IdentifiableObjectUtils.java 2012-12-22 17:51:57 +0000
@@ -65,6 +65,24 @@
return null;
}
+
+ /**
+ * Returns a list of uids for the given collection of IdentifiableObjects.
+ *
+ * @param objects the list of IdentifiableObjects.
+ * @return a list of uids.
+ */
+ public static <T extends IdentifiableObject> List<String> getUids( Collection<T> objects )
+ {
+ List<String> uids = new ArrayList<String>();
+
+ for ( T object : objects )
+ {
+ uids.add( object.getUid() );
+ }
+
+ return uids;
+ }
/**
* Filters the given list of IdentifiableObjects based on the given key.
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java 2012-12-22 12:26:34 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java 2012-12-22 17:51:57 +0000
@@ -27,6 +27,7 @@
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
+import java.util.Collection;
import java.util.Date;
import java.util.List;
import java.util.concurrent.Future;
@@ -107,4 +108,15 @@
* @param tableName the name of the table to drop.
*/
void dropTable( String tableName );
+
+ /**
+ * Applies aggregation level logic to the analytics table by setting the
+ * organisation unit level column values to null for the levels above the
+ * given aggregation level.
+ *
+ * @param tableName the name of the analytics table.
+ * @param dataElements the data element uids to apply aggregation levels for.
+ * @param aggregationLevel the aggregation level.
+ */
+ void applyAggregationLevels( String tableName, Collection<String> dataElements, int aggregationLevel );
}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java 2012-12-22 12:26:34 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java 2012-12-22 17:51:57 +0000
@@ -28,6 +28,7 @@
*/
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
@@ -37,6 +38,9 @@
import org.apache.commons.logging.LogFactory;
import org.hisp.dhis.analytics.AnalyticsTableManager;
import org.hisp.dhis.analytics.AnalyticsTableService;
+import org.hisp.dhis.common.IdentifiableObjectUtils;
+import org.hisp.dhis.dataelement.DataElementService;
+import org.hisp.dhis.organisationunit.OrganisationUnitService;
import org.hisp.dhis.period.Period;
import org.hisp.dhis.system.util.Clock;
import org.hisp.dhis.system.util.ConcurrentUtils;
@@ -52,6 +56,12 @@
@Autowired
private AnalyticsTableManager tableManager;
+
+ @Autowired
+ private OrganisationUnitService organisationUnitService;
+
+ @Autowired
+ private DataElementService dataElementService;
// -------------------------------------------------------------------------
// Implementation
@@ -82,6 +92,9 @@
pruneTables( tables );
clock.logTime( "Pruned analytics tables" );
+ applyAggregationLevels( tables );
+ clock.logTime( "Applied aggregation levels" );
+
createIndexes( tables );
clock.logTime( "Created all indexes" );
@@ -130,7 +143,7 @@
}
}
- public void pruneTables( List<String> tables )
+ private void pruneTables( List<String> tables )
{
Iterator<String> iterator = tables.iterator();
@@ -143,6 +156,27 @@
}
}
+ private void applyAggregationLevels( List<String> tables )
+ {
+ int maxLevels = organisationUnitService.getMaxOfOrganisationUnitLevels();
+
+ for ( int i = 0; i < maxLevels; i++ )
+ {
+ int level = maxLevels - i;
+
+ Collection<String> dataElements = IdentifiableObjectUtils.getUids(
+ dataElementService.getDataElementsByAggregationLevel( level ) );
+
+ if ( !dataElements.isEmpty() )
+ {
+ for ( String table : tables )
+ {
+ tableManager.applyAggregationLevels( table, dataElements, level );
+ }
+ }
+ }
+ }
+
private void createIndexes( List<String> tables )
{
int pages = Math.max( ( SystemUtils.getCpuCores() - 1 ), 1 );
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java 2012-12-22 12:26:34 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java 2012-12-22 17:51:57 +0000
@@ -37,6 +37,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.hisp.dhis.analytics.AnalyticsTableManager;
+import org.hisp.dhis.analytics.DataQueryParams;
import org.hisp.dhis.common.CodeGenerator;
import org.hisp.dhis.dataelement.DataElementGroupSet;
import org.hisp.dhis.dataelement.DataElementService;
@@ -51,6 +52,8 @@
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.scheduling.annotation.Async;
+import static org.hisp.dhis.system.util.TextUtils.getQuotedCommaDelimitedString;
+
/**
* This class manages the analytics table. The analytics table is a denormalized
* table designed for analysis which contains raw data values. It has columns for
@@ -249,8 +252,9 @@
String[] de = { "de", "character(11) not null", "de.uid" };
String[] co = { "coc", "character(11) not null", "coc.uid" };
+ String[] level = { "level", "integer", "ous.level" };
- columns.addAll( Arrays.asList( de, co ) );
+ columns.addAll( Arrays.asList( de, co, level ) );
return columns;
}
@@ -315,6 +319,29 @@
executeSilently( "drop table " + realTable );
}
+ public void applyAggregationLevels( String tableName, Collection<String> dataElements, int aggregationLevel )
+ {
+ StringBuilder sql = new StringBuilder( "update " + tableName + " set " );
+
+ for ( int i = 0; i < aggregationLevel; i++ )
+ {
+ int level = i + 1;
+
+ String column = DataQueryParams.LEVEL_PREFIX + level;
+
+ sql.append( column + " = null," );
+ }
+
+ sql.deleteCharAt( sql.length() - ",".length() );
+
+ sql.append( " where level > " + aggregationLevel );
+ sql.append( " and de in (" + getQuotedCommaDelimitedString( dataElements ) + ")" );
+
+ log.info( "Aggregation level SQL: " + sql.toString() );
+
+ jdbcTemplate.execute( sql.toString() );
+ }
+
// -------------------------------------------------------------------------
// Supportive methods
// -------------------------------------------------------------------------
