dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #27890
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 13893: Analytics, added analytics table for calculating no org units in groups at levels/in group sets
------------------------------------------------------------
revno: 13893
committer: Lars Helge Øverland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Thu 2014-01-30 12:13:58 +0200
message:
Analytics, added analytics table for calculating no org units in groups at levels/in group sets
added:
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java
modified:
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/scheduling/AnalyticsTableTask.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java
dhis-2/dhis-services/dhis-service-analytics/src/main/resources/META-INF/dhis/beans.xml
dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/DefaultExpressionService.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java 2013-08-23 16:05:01 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java 2014-01-30 10:13:58 +0000
@@ -41,8 +41,9 @@
{
public static final String TABLE_TEMP_SUFFIX = "_temp";
public static final String ANALYTICS_TABLE_NAME = "analytics";
- public static final String COMPLETENESS_TABLE_NAME = "completeness";
- public static final String COMPLETENESS_TARGET_TABLE_NAME = "completenesstarget";
+ public static final String COMPLETENESS_TABLE_NAME = "analytics_completeness";
+ public static final String COMPLETENESS_TARGET_TABLE_NAME = "analytics_completenesstarget";
+ public static final String ORGUNIT_TARGET_TABLE_NAME = "analytics_orgunittarget";
/**
* Returns analytics tables which yearly partitions. Yearly partitions will
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/scheduling/AnalyticsTableTask.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/scheduling/AnalyticsTableTask.java 2013-12-25 21:54:14 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/scheduling/AnalyticsTableTask.java 2014-01-30 10:13:58 +0000
@@ -54,6 +54,9 @@
@Resource(name="org.hisp.dhis.analytics.CompletenessTargetTableService")
private AnalyticsTableService completenessTargetTableService;
+ @Resource(name="org.hisp.dhis.analytics.OrgUnitTargetTableService")
+ private AnalyticsTableService orgUnitTargetTableService;
+
@Resource(name="org.hisp.dhis.analytics.EventAnalyticsTableService")
private AnalyticsTableService eventAnalyticsTableService;
@@ -105,33 +108,38 @@
@Override
public void run()
{
- notifier.clear( taskId ).notify( taskId, "Updating resource tables" );
+ notifier.clear( taskId ).notify( taskId, "Analytics table update process started" );
try
{
if ( !skipResourceTables )
{
+ notifier.notify( taskId, "Updating resource tables" );
analyticsTableService.generateResourceTables();
+ }
+
+ if ( !skipAggregate )
+ {
notifier.notify( taskId, "Updating analytics tables" );
- }
-
- if ( !skipAggregate )
- {
analyticsTableService.update( last3Years, taskId );
- notifier.notify( taskId, "Updating completeness tables" );
-
+
+ notifier.notify( taskId, "Updating completeness table" );
completenessTableService.update( last3Years, taskId );
- notifier.notify( taskId, "Updating compeleteness target table" );
-
- completenessTargetTableService.update( last3Years, taskId );
- notifier.notify( taskId, "Updating event analytics tables" );
+
+ notifier.notify( taskId, "Updating completeness target table" );
+ completenessTargetTableService.update( last3Years, taskId );
+
+ notifier.notify( taskId, "Updating organisation unit target table" );
+ orgUnitTargetTableService.update( last3Years, taskId );
}
if ( !skipEvents )
{
+ notifier.notify( taskId, "Updating event analytics table" );
eventAnalyticsTableService.update( last3Years, taskId );
- notifier.notify( taskId, INFO, "Analytics tables updated", true );
}
+
+ notifier.notify( taskId, INFO, "Analytics tables updated", true );
}
catch ( RuntimeException ex )
{
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java 2014-01-17 14:57:16 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java 2014-01-30 10:13:58 +0000
@@ -80,7 +80,7 @@
public String getTableName()
{
- return "analytics";
+ return ANALYTICS_TABLE_NAME;
}
public void createTable( AnalyticsTable table )
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java 2013-12-30 11:25:14 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java 2014-01-30 10:13:58 +0000
@@ -55,7 +55,7 @@
public String getTableName()
{
- return "completeness";
+ return COMPLETENESS_TABLE_NAME;
}
public void createTable( AnalyticsTable table )
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java 2013-12-30 11:25:14 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java 2014-01-30 10:13:58 +0000
@@ -63,7 +63,7 @@
public String getTableName()
{
- return "completenesstarget";
+ return COMPLETENESS_TARGET_TABLE_NAME;
}
public void createTable( AnalyticsTable table )
=== added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java 1970-01-01 00:00:00 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java 2014-01-30 10:13:58 +0000
@@ -0,0 +1,179 @@
+package org.hisp.dhis.analytics.table;
+
+/*
+ * Copyright (c) 2004-2013, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.Future;
+
+import org.hisp.dhis.analytics.AnalyticsTable;
+import org.hisp.dhis.organisationunit.OrganisationUnitGroupSet;
+import org.hisp.dhis.organisationunit.OrganisationUnitLevel;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.transaction.annotation.Transactional;
+
+/**
+ * @author Lars Helge Overland
+ */
+public class JdbcOrgUnitTargetTableManager
+ extends AbstractJdbcTableManager
+{
+ @Override
+ @Transactional
+ public List<AnalyticsTable> getTables( boolean last3YearsOnly )
+ {
+ List<AnalyticsTable> tables = new ArrayList<AnalyticsTable>();
+ tables.add( new AnalyticsTable( getTableName(), getDimensionColumns( null ) ) );
+ return tables;
+ }
+
+ public boolean validState()
+ {
+ return true;
+ }
+
+ public String getTableName()
+ {
+ return ORGUNIT_TARGET_TABLE_NAME;
+ }
+
+ public void createTable( AnalyticsTable table )
+ {
+ final String tableName = table.getTempTableName();
+
+ final String sqlDrop = "drop table " + tableName;
+
+ executeSilently( sqlDrop );
+
+ String sqlCreate = "create table " + tableName + " (";
+
+ for ( String[] col : getDimensionColumns( table ) )
+ {
+ sqlCreate += col[0] + " " + col[1] + ",";
+ }
+
+ sqlCreate += "value double precision) ";
+
+ sqlCreate += statementBuilder.getTableOptions( false );
+
+ log.info( "Create SQL: " + sqlCreate );
+
+ executeSilently( sqlCreate );
+ }
+
+ @Async
+ public Future<?> populateTableAsync( ConcurrentLinkedQueue<AnalyticsTable> tables )
+ {
+ taskLoop : while ( true )
+ {
+ AnalyticsTable table = tables.poll();
+
+ if ( table == null )
+ {
+ break taskLoop;
+ }
+
+ String sql = "insert into " + table.getTempTableName() + " (";
+
+ for ( String[] col : getDimensionColumns( table ) )
+ {
+ sql += col[0] + ",";
+ }
+
+ sql += "value) select ";
+
+ for ( String[] col : getDimensionColumns( table ) )
+ {
+ sql += col[2] + ",";
+ }
+
+ sql +=
+ "1 as value " +
+ "from orgunitgroupmembers ougm " +
+ "left join orgunitgroup oug on ougm.orgunitgroupid=oug.orgunitgroupid " +
+ "left join _orgunitstructure ous on ougm.organisationunitid=ous.organisationunitid " +
+ "left join _organisationunitgroupsetstructure ougs on ougm.organisationunitid=ougs.organisationunitid";
+
+ log.info( "Populate SQL: "+ sql );
+
+ jdbcTemplate.execute( sql );
+ }
+
+ return null;
+ }
+
+ public List<String[]> getDimensionColumns( AnalyticsTable table )
+ {
+ List<String[]> columns = new ArrayList<String[]>();
+
+ Collection<OrganisationUnitGroupSet> orgUnitGroupSets =
+ organisationUnitGroupService.getDataDimensionOrganisationUnitGroupSets();
+
+ Collection<OrganisationUnitLevel> levels =
+ organisationUnitService.getOrganisationUnitLevels();
+
+ for ( OrganisationUnitGroupSet groupSet : orgUnitGroupSets )
+ {
+ String[] col = { quote( groupSet.getUid() ), "character(11)", "ougs." + quote( groupSet.getUid() ) };
+ columns.add( col );
+ }
+
+ for ( OrganisationUnitLevel level : levels )
+ {
+ String column = quote( PREFIX_ORGUNITLEVEL + level.getLevel() );
+ String[] col = { column, "character(11)", "ous." + column };
+ columns.add( col );
+ }
+
+ String[] ds = { "oug", "character(11) not null", "oug.uid" };
+
+ columns.add( ds );
+
+ return columns;
+ }
+
+ public Date getEarliestData()
+ {
+ return null; // Not relevant
+ }
+
+ public Date getLatestData()
+ {
+ return null; // Not relevant
+ }
+
+ @Async
+ public Future<?> applyAggregationLevels( ConcurrentLinkedQueue<AnalyticsTable> tables, Collection<String> dataElements, int aggregationLevel )
+ {
+ return null; // Not relevant
+ }
+}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/resources/META-INF/dhis/beans.xml'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/resources/META-INF/dhis/beans.xml 2013-09-29 15:41:22 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/resources/META-INF/dhis/beans.xml 2014-01-30 10:13:58 +0000
@@ -10,6 +10,8 @@
<bean id="org.hisp.dhis.analytics.CompletenessTargetTableManager" class="org.hisp.dhis.analytics.table.JdbcCompletenessTargetTableManager" />
+ <bean id="org.hisp.dhis.analytics.OrgUnitTargetTableManager" class="org.hisp.dhis.analytics.table.JdbcOrgUnitTargetTableManager" />
+
<bean id="org.hisp.dhis.analytics.EventAnalyticsTableManager" class="org.hisp.dhis.analytics.table.JdbcEventAnalyticsTableManager" />
<bean id="org.hisp.dhis.analytics.AnalyticsTableService" class="org.hisp.dhis.analytics.table.DefaultAnalyticsTableService">
@@ -24,6 +26,10 @@
<property name="tableManager" ref="org.hisp.dhis.analytics.CompletenessTargetTableManager" />
</bean>
+ <bean id="org.hisp.dhis.analytics.OrgUnitTargetTableService" class="org.hisp.dhis.analytics.table.DefaultAnalyticsTableService">
+ <property name="tableManager" ref="org.hisp.dhis.analytics.OrgUnitTargetTableManager" />
+ </bean>
+
<bean id="org.hisp.dhis.analytics.EventAnalyticsTableService" class="org.hisp.dhis.analytics.table.DefaultAnalyticsTableService">
<property name="tableManager" ref="org.hisp.dhis.analytics.EventAnalyticsTableManager" />
</bean>
=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/DefaultExpressionService.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/DefaultExpressionService.java 2013-12-01 16:19:14 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/DefaultExpressionService.java 2014-01-30 10:13:58 +0000
@@ -183,22 +183,24 @@
return null;
}
-
- public Double getExpressionValue( Expression expression, Map<DataElementOperand, Double> valueMap,
- Map<String, Double> constantMap, Integer days )
- {
- final String expressionString = generateExpression( expression.getExpression(), valueMap, constantMap, days, expression.isNullIfBlank() );
-
- return expressionString != null ? calculateExpression( expressionString ) : null;
- }
-
- public Double getExpressionValue( Expression expression, Map<DataElementOperand, Double> valueMap,
- Map<String, Double> constantMap, Integer days, Set<DataElementOperand> incompleteValues )
- {
- final String expressionString = generateExpression( expression.getExpression(), valueMap, constantMap, days, expression.isNullIfBlank(), incompleteValues );
-
- return expressionString != null ? calculateExpression( expressionString ) : null;
- }
+
+ public Double getExpressionValue( Expression expression, Map<DataElementOperand, Double> valueMap,
+ Map<String, Double> constantMap, Integer days )
+ {
+ final String expressionString = generateExpression( expression.getExpression(), valueMap, constantMap, days,
+ expression.isNullIfBlank() );
+
+ return expressionString != null ? calculateExpression( expressionString ) : null;
+ }
+
+ public Double getExpressionValue( Expression expression, Map<DataElementOperand, Double> valueMap,
+ Map<String, Double> constantMap, Integer days, Set<DataElementOperand> incompleteValues )
+ {
+ final String expressionString = generateExpression( expression.getExpression(), valueMap, constantMap, days,
+ expression.isNullIfBlank(), incompleteValues );
+
+ return expressionString != null ? calculateExpression( expressionString ) : null;
+ }
@Transactional
public Set<DataElement> getDataElementsInExpression( String expression )
@@ -647,13 +649,14 @@
}
@Transactional
- public String generateExpression( String expression, Map<DataElementOperand, Double> valueMap, Map<String, Double> constantMap, Integer days, boolean nullIfNoValues )
+ public String generateExpression( String expression, Map<DataElementOperand, Double> valueMap,
+ Map<String, Double> constantMap, Integer days, boolean nullIfNoValues )
{
return generateExpression( expression, valueMap, constantMap, days, nullIfNoValues, null );
}
- private String generateExpression( String expression, Map<DataElementOperand, Double> valueMap, Map<String, Double> constantMap, Integer days, boolean nullIfNoValues,
- Set<DataElementOperand> incompleteValues )
+ private String generateExpression( String expression, Map<DataElementOperand, Double> valueMap,
+ Map<String, Double> constantMap, Integer days, boolean nullIfNoValues, Set<DataElementOperand> incompleteValues )
{
if ( expression == null || expression.isEmpty() )
{
