dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #21178
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 9951: Analytics, user feedback during table generation wip
------------------------------------------------------------
revno: 9951
committer: Lars Helge Øverland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Sun 2013-03-03 22:10:52 +0100
message:
Analytics, user feedback during table generation wip
modified:
dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/TaskCategory.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableService.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java
dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/InMemoryNotifier.java
dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/api/controller/ResourceTableController.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/TaskCategory.java'
--- dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/TaskCategory.java 2012-06-27 15:59:59 +0000
+++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/TaskCategory.java 2013-03-03 21:10:52 +0000
@@ -33,6 +33,7 @@
public enum TaskCategory
{
DATAMART,
+ ANALYTICS_UPDATE,
DATAVALUE_IMPORT,
METADATA_IMPORT,
METADATA_EXPORT
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableService.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableService.java 2012-12-03 21:04:39 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableService.java 2013-03-03 21:10:52 +0000
@@ -29,7 +29,14 @@
import java.util.concurrent.Future;
+import org.hisp.dhis.scheduling.TaskId;
+
public interface AnalyticsTableService
{
- Future<?> update();
+ /**
+ * Rebuilds the analytics tables.
+ *
+ * @param taskId the TaskId.
+ */
+ Future<?> update( TaskId taskId );
}
=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java 2013-02-19 11:54:49 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java 2013-03-03 21:10:52 +0000
@@ -43,12 +43,17 @@
import org.hisp.dhis.common.IdentifiableObjectUtils;
import org.hisp.dhis.dataelement.DataElementService;
import org.hisp.dhis.organisationunit.OrganisationUnitService;
+import org.hisp.dhis.scheduling.TaskId;
+import org.hisp.dhis.system.notification.Notifier;
import org.hisp.dhis.system.util.Clock;
import org.hisp.dhis.system.util.ConcurrentUtils;
+import org.hisp.dhis.system.util.DebugUtils;
import org.hisp.dhis.system.util.SystemUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Async;
+import static org.hisp.dhis.scheduling.TaskCategory.ANALYTICS_UPDATE;
+
public class DefaultAnalyticsTableService
implements AnalyticsTableService
{
@@ -66,50 +71,71 @@
@Autowired
private DataElementService dataElementService;
+
+ @Autowired
+ private Notifier notifier;
// -------------------------------------------------------------------------
// Implementation
// -------------------------------------------------------------------------
-
- //TODO generateOrganisationUnitStructures
- // generateOrganisationUnitGroupSetTable
- // generatePeriodStructure
@Async
- public Future<?> update()
+ public Future<?> update( TaskId taskId )
{
- Clock clock = new Clock().startClock().logTime( "Starting update..." );
-
- final Date earliest = tableManager.getEarliestData();
- final Date latest = tableManager.getLatestData();
- final String tableName = tableManager.getTableName();
- final List<String> tables = PartitionUtils.getTempTableNames( earliest, latest, tableName );
- clock.logTime( "Got partition tables: " + tables + ", earliest: " + earliest + ", latest: " + latest );
-
- //dropTables( tables );
-
- createTables( tables );
- clock.logTime( "Created analytics tables" );
-
- populateTables( tables );
- clock.logTime( "Populated analytics tables" );
-
- pruneTables( tables );
- clock.logTime( "Pruned analytics tables" );
-
- applyAggregationLevels( tables );
- clock.logTime( "Applied aggregation levels" );
-
- createIndexes( tables );
- clock.logTime( "Created all indexes" );
-
- vacuumTables( tables );
- clock.logTime( "Vacuumed tables" );
-
- swapTables( tables );
- clock.logTime( "Swapped analytics tables" );
-
- clock.logTime( "Analytics tables update done" );
+ Clock clock = new Clock().startClock().logTime( "Starting update" );
+
+ try
+ {
+ final Date earliest = tableManager.getEarliestData();
+ final Date latest = tableManager.getLatestData();
+ final String tableName = tableManager.getTableName();
+ final List<String> tables = PartitionUtils.getTempTableNames( earliest, latest, tableName );
+ clock.logTime( "Got partition tables: " + tables + ", earliest: " + earliest + ", latest: " + latest );
+
+ notifier.notify( taskId, ANALYTICS_UPDATE, "Creating analytics tables" );
+
+ createTables( tables );
+
+ clock.logTime( "Created analytics tables" );
+ notifier.notify( taskId, ANALYTICS_UPDATE, "Populating analytics tables" );
+
+ populateTables( tables );
+
+ clock.logTime( "Populated analytics tables" );
+ notifier.notify( taskId, ANALYTICS_UPDATE, "Pruned analytics tables" );
+
+ pruneTables( tables );
+
+ clock.logTime( "Pruned analytics tables" );
+ notifier.notify( taskId, ANALYTICS_UPDATE, "Applying aggregation levels" );
+
+ applyAggregationLevels( tables );
+
+ clock.logTime( "Applied aggregation levels" );
+ notifier.notify( taskId, ANALYTICS_UPDATE, "Creating indexes" );
+
+ createIndexes( tables );
+
+ clock.logTime( "Created indexes" );
+ notifier.notify( taskId, ANALYTICS_UPDATE, "Vacuuming tables" );
+
+ vacuumTables( tables );
+
+ clock.logTime( "Vacuumed tables" );
+ notifier.notify( taskId, ANALYTICS_UPDATE, "Swapping analytics tables" );
+
+ swapTables( tables );
+
+ clock.logTime( "Swapped analytics tables" );
+
+ clock.logTime( "Table update done" );
+ notifier.notify( taskId, ANALYTICS_UPDATE, "Table update done" );
+ }
+ catch ( Exception ex )
+ {
+ log.error( "Error during analytics table generation", ex );
+ log.error( DebugUtils.getStackTrace( ex ) );
+ }
return null;
}
=== modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/InMemoryNotifier.java'
--- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/InMemoryNotifier.java 2012-06-03 15:04:33 +0000
+++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/InMemoryNotifier.java 2013-03-03 21:10:52 +0000
@@ -77,11 +77,14 @@
{
Notification notification = new Notification( level, category, new Date(), message, completed );
- notifications.get( id ).add( 0, notification );
-
- if ( notifications.get( id ).size() > MAX_SIZE )
+ if ( id != null )
{
- notifications.get( id ).remove( MAX_SIZE );
+ notifications.get( id ).add( 0, notification );
+
+ if ( notifications.get( id ).size() > MAX_SIZE )
+ {
+ notifications.get( id ).remove( MAX_SIZE );
+ }
}
log.info( notification );
=== modified file 'dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/api/controller/ResourceTableController.java'
--- dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/api/controller/ResourceTableController.java 2013-03-01 17:59:05 +0000
+++ dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/api/controller/ResourceTableController.java 2013-03-03 21:10:52 +0000
@@ -33,7 +33,8 @@
import org.hisp.dhis.analytics.AnalyticsTableService;
import org.hisp.dhis.api.utils.ContextUtils;
import org.hisp.dhis.resourcetable.ResourceTableService;
-import org.hisp.dhis.sqlview.SqlViewService;
+import org.hisp.dhis.system.scheduling.Scheduler;
+import org.hisp.dhis.user.CurrentUserService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Controller;
@@ -60,15 +61,18 @@
@Autowired
private ResourceTableService resourceTableService;
-
- @Autowired
- private SqlViewService sqlViewService;
-
+
+ @Autowired
+ private CurrentUserService currentUserService;
+
+ @Autowired
+ private Scheduler scheduler;
+
@RequestMapping( value = "/analytics", method = RequestMethod.PUT )
@PreAuthorize( "hasRole('ALL') or hasRole('F_DATA_MART_ADMIN')" )
- public void analytics( HttpServletResponse response )
+ public void data( HttpServletResponse response )
{
- analyticsTableService.update();
+ analyticsTableService.update( null );
ContextUtils.okResponse( response, "Initiated analytics table update" );
}
@@ -77,7 +81,7 @@
@PreAuthorize( "hasRole('ALL') or hasRole('F_DATA_MART_ADMIN')" )
public void completeness( HttpServletResponse response )
{
- completenessTableService.update();
+ completenessTableService.update( null );
ContextUtils.okResponse( response, "Initiated completeness table update" );
}
@@ -86,7 +90,7 @@
@PreAuthorize( "hasRole('ALL') or hasRole('F_DATA_MART_ADMIN')" )
public void completenessTarget( HttpServletResponse response )
{
- completenessTargetTableService.update();
+ completenessTargetTableService.update( null );
ContextUtils.okResponse( response, "Initiated completeness target table update" );
}
