dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #31550
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 16091: Impl scheduling of data value synch. Data synchs every minute.
------------------------------------------------------------
revno: 16091
committer: Lars Helge Overland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Fri 2014-07-11 23:22:57 +0200
message:
Impl scheduling of data value synch. Data synchs every minute.
modified:
dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/synch/DefaultSynchronizationManager.java
dhis-2/dhis-services/dhis-service-dxf2/src/main/resources/META-INF/dhis/beans.xml
dhis-2/dhis-services/dhis-service-reporting/pom.xml
dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/scheduling/DefaultSchedulingManager.java
dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/scheduling/SchedulingManager.java
dhis-2/dhis-services/dhis-service-reporting/src/main/resources/META-INF/dhis/beans.xml
dhis-2/dhis-services/dhis-service-reporting/src/test/java/org/hisp/dhis/scheduling/SchedulingManagerTest.java
dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/Scheduler.java
dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/java/org/hisp/dhis/dataadmin/action/scheduling/ScheduleTasksAction.java
dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/resources/org/hisp/dhis/dataadmin/i18n_module.properties
dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/webapp/dhis-web-maintenance-dataadmin/javascript/scheduling.js
dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/webapp/dhis-web-maintenance-dataadmin/viewScheduledTasks.vm
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/synch/DefaultSynchronizationManager.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/synch/DefaultSynchronizationManager.java 2014-07-11 15:50:18 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/synch/DefaultSynchronizationManager.java 2014-07-11 21:22:57 +0000
@@ -112,6 +112,7 @@
ResponseEntity<String> response = null;
HttpStatus sc = null;
+ String st = null;
AvailabilityStatus status = null;
try
@@ -122,10 +123,12 @@
catch ( HttpClientErrorException ex )
{
sc = ex.getStatusCode();
+ st = ex.getStatusText();
}
catch ( HttpServerErrorException ex )
{
sc = ex.getStatusCode();
+ st = ex.getStatusText();
}
catch( ResourceAccessException ex )
{
@@ -136,15 +139,15 @@
if ( HttpStatus.FOUND.equals( sc ) )
{
- status = new AvailabilityStatus( false, "Server is available but no authentication was provided" );
+ status = new AvailabilityStatus( false, "Server is available but no authentication was provided, status code: " + sc );
}
else if ( HttpStatus.UNAUTHORIZED.equals( sc ) )
{
- status = new AvailabilityStatus( false, "Server is available but authentication failed" );
+ status = new AvailabilityStatus( false, "Server is available but authentication failed, status code: " + sc );
}
else if ( HttpStatus.INTERNAL_SERVER_ERROR.equals( sc ) )
{
- status = new AvailabilityStatus( false, "Server is available but experienced an internal error" );
+ status = new AvailabilityStatus( false, "Server is available but experienced an internal error, status code: " + sc );
}
else if ( HttpStatus.OK.equals( sc ) )
{
@@ -152,7 +155,7 @@
}
else
{
- status = new AvailabilityStatus( false, "Server is not available for unknown reason: " + sc );
+ status = new AvailabilityStatus( false, "Server is not available, status code: " + sc + ", text: " + st );
}
log.info( status );
@@ -176,13 +179,14 @@
// subsequently part of next synch process without being ignored
// ---------------------------------------------------------------------
- final Date time = getLastSynchSuccess();
+ final Date startTime = new Date();
+ final Date lastSuccessTime = getLastSynchSuccess();
- int lastUpdatedCount = dataValueService.getDataValueCountLastUpdatedAfter( time );
+ int lastUpdatedCount = dataValueService.getDataValueCountLastUpdatedAfter( lastSuccessTime );
if ( lastUpdatedCount == 0 )
{
- log.info( "Aborting synch, no new or updated data values" );
+ log.info( "Skipping synch, no new or updated data values" );
return null;
}
@@ -198,7 +202,7 @@
{
request.getHeaders().setContentType( MediaType.APPLICATION_JSON );
request.getHeaders().add( HEADER_AUTHORIZATION, CodecUtils.getBasicAuthString( config.getRemoteServerUsername(), config.getRemoteServerPassword() ) );
- dataValueSetService.writeDataValueSetJson( time, request.getBody() );
+ dataValueSetService.writeDataValueSetJson( lastSuccessTime, request.getBody() );
}
};
@@ -210,8 +214,8 @@
if ( summary != null && ImportStatus.SUCCESS.equals( summary.getStatus() ) )
{
- setLastSynchSuccess( time );
- log.info( "Synch successful, setting last success time: " + time );
+ setLastSynchSuccess( startTime );
+ log.info( "Synch successful, setting last success time: " + startTime );
}
return summary;
@@ -227,9 +231,7 @@
*/
private Date getLastSynchSuccess()
{
- Date date = (Date) systemSettingManager.getSystemSetting( KEY_LAST_SUCCESSFUL_SYNC );
-
- return date != null ? date : new Date();
+ return (Date) systemSettingManager.getSystemSetting( KEY_LAST_SUCCESSFUL_SYNC, new Date() );
}
/**
=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/resources/META-INF/dhis/beans.xml'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/resources/META-INF/dhis/beans.xml 2014-07-11 15:50:18 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/resources/META-INF/dhis/beans.xml 2014-07-11 21:22:57 +0000
@@ -39,7 +39,7 @@
<bean id="org.hisp.dhis.dxf2.synch.SynchronizationManager" class="org.hisp.dhis.dxf2.synch.DefaultSynchronizationManager" />
- <bean id="dataSynchronizationTask" class="org.hisp.dhis.dxf2.synch.DataSynchronizationTask" />
+ <bean id="dataSynchTask" class="org.hisp.dhis.dxf2.synch.DataSynchronizationTask" />
<!-- register idObject handlers -->
=== modified file 'dhis-2/dhis-services/dhis-service-reporting/pom.xml'
--- dhis-2/dhis-services/dhis-service-reporting/pom.xml 2014-07-06 15:43:19 +0000
+++ dhis-2/dhis-services/dhis-service-reporting/pom.xml 2014-07-11 21:22:57 +0000
@@ -40,7 +40,11 @@
<dependency>
<groupId>org.hisp.dhis</groupId>
<artifactId>dhis-service-eventreporting</artifactId>
- </dependency>
+ </dependency>
+ <dependency>
+ <groupId>org.hisp.dhis</groupId>
+ <artifactId>dhis-service-dxf2</artifactId>
+ </dependency>
<!-- JasperReports -->
=== modified file 'dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/scheduling/DefaultSchedulingManager.java'
--- dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/scheduling/DefaultSchedulingManager.java 2014-03-18 08:10:10 +0000
+++ dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/scheduling/DefaultSchedulingManager.java 2014-07-11 21:22:57 +0000
@@ -32,13 +32,18 @@
import static org.hisp.dhis.system.scheduling.Scheduler.STATUS_NOT_STARTED;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
import org.hisp.dhis.common.ListMap;
import org.hisp.dhis.setting.SystemSettingManager;
import org.hisp.dhis.system.scheduling.Scheduler;
/**
+ * Cron refers to the cron expression used for scheduling. Key refers to the key
+ * identifying the scheduled tasks.
+ *
* @author Lars Helge Overland
*/
public class DefaultSchedulingManager
@@ -110,6 +115,20 @@
return (ListMap<String, String>) systemSettingManager.getSystemSetting( KEY_SCHEDULED_TASKS, new ListMap<String, String>() );
}
+ public Set<String> getScheduledKeys()
+ {
+ ListMap<String, String> cronKeyMap = getCronKeyMap();
+
+ Set<String> keys = new HashSet<>();
+
+ for ( String cron : cronKeyMap.keySet() )
+ {
+ keys.addAll( cronKeyMap.get( cron ) );
+ }
+
+ return keys;
+ }
+
public String getTaskStatus()
{
ListMap<String, String> cronKeyMap = getCronKeyMap();
=== modified file 'dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/scheduling/SchedulingManager.java'
--- dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/scheduling/SchedulingManager.java 2014-07-11 19:12:01 +0000
+++ dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/scheduling/SchedulingManager.java 2014-07-11 21:22:57 +0000
@@ -28,6 +28,8 @@
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
+import java.util.Set;
+
import org.hisp.dhis.common.ListMap;
/**
@@ -40,31 +42,37 @@
final String TASK_ANALYTICS_ALL = "analyticsAllTask";
final String TASK_ANALYTICS_LAST_3_YEARS = "analyticsLast3YearsTask";
final String TASK_MONITORING_LAST_DAY = "monitoringLastDayTask";
-
+ final String TASK_DATA_SYNCH = "dataSynchTask";
+
/**
- * Schedule all tasks.
+ * Schedules all tasks.
*/
void scheduleTasks();
/**
- * Schedule the given tasks.
+ * Schedules the given tasks.
*
* @param cronKeyMap a mapping of cron expressions and task keys.
*/
void scheduleTasks( ListMap<String, String> cronKeyMap );
/**
- * Stop all tasks.
+ * Stops all tasks.
*/
void stopTasks();
/**
- * Get a mapping of cron expressions and list of task keys for all scheduled
+ * Gets a mapping of cron expressions and list of task keys for all scheduled
* tasks.
*/
ListMap<String, String> getCronKeyMap();
/**
+ * Gets all keys currenty scheduled for any task.
+ */
+ Set<String> getScheduledKeys();
+
+ /**
* Gets the task status. Can be STATUS_RUNNING, STATUS_DONE, STATUS_STOPPED,
* STATUS_NOT_STARTED.
*/
=== modified file 'dhis-2/dhis-services/dhis-service-reporting/src/main/resources/META-INF/dhis/beans.xml'
--- dhis-2/dhis-services/dhis-service-reporting/src/main/resources/META-INF/dhis/beans.xml 2014-07-11 19:12:01 +0000
+++ dhis-2/dhis-services/dhis-service-reporting/src/main/resources/META-INF/dhis/beans.xml 2014-07-11 21:22:57 +0000
@@ -191,6 +191,7 @@
<entry key="analyticsAllTask" value-ref="analyticsAllTask" />
<entry key="analyticsLast3YearsTask" value-ref="analyticsLast3YearsTask" />
<entry key="monitoringLastDayTask" value-ref="monitoringLastDayTask" />
+ <entry key="dataSynchTask" value-ref="dataSynchTask" />
</map>
</property>
</bean>
=== modified file 'dhis-2/dhis-services/dhis-service-reporting/src/test/java/org/hisp/dhis/scheduling/SchedulingManagerTest.java'
--- dhis-2/dhis-services/dhis-service-reporting/src/test/java/org/hisp/dhis/scheduling/SchedulingManagerTest.java 2014-07-11 19:12:01 +0000
+++ dhis-2/dhis-services/dhis-service-reporting/src/test/java/org/hisp/dhis/scheduling/SchedulingManagerTest.java 2014-07-11 21:22:57 +0000
@@ -35,6 +35,8 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.util.Set;
+
import org.hisp.dhis.DhisSpringTest;
import org.hisp.dhis.common.ListMap;
import org.hisp.dhis.system.scheduling.Scheduler;
@@ -86,4 +88,22 @@
assertEquals( Scheduler.STATUS_NOT_STARTED, schedulingManager.getTaskStatus() );
}
+
+ @Test
+ public void testGetScheduledKeys()
+ {
+ ListMap<String, String> cronKeyMap = new ListMap<String, String>();
+ cronKeyMap.putValue( CRON_DAILY_0AM, TASK_RESOURCE_TABLE );
+ cronKeyMap.putValue( CRON_DAILY_0AM, TASK_ANALYTICS_ALL );
+ cronKeyMap.putValue( CRON_DAILY_0AM, TASK_DATAMART_LAST_YEAR );
+
+ schedulingManager.scheduleTasks( cronKeyMap );
+
+ Set<String> keys = schedulingManager.getScheduledKeys();
+
+ assertEquals( 3, keys.size() );
+ assertTrue( keys.contains( TASK_RESOURCE_TABLE ) );
+ assertTrue( keys.contains( TASK_ANALYTICS_ALL ) );
+ assertTrue( keys.contains( TASK_DATAMART_LAST_YEAR ) );
+ }
}
=== modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/Scheduler.java'
--- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/Scheduler.java 2014-07-11 19:12:01 +0000
+++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/Scheduler.java 2014-07-11 21:22:57 +0000
@@ -34,6 +34,7 @@
public interface Scheduler
{
final String CRON_DAILY_0AM = "0 0 0 * * ?";
+ final String CRON_EVERY_MIN = "0 0/1 * * * ?";
final String CRON_TEST = "0 * * * * ?";
final String STATUS_RUNNING = "running";
=== modified file 'dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/java/org/hisp/dhis/dataadmin/action/scheduling/ScheduleTasksAction.java'
--- dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/java/org/hisp/dhis/dataadmin/action/scheduling/ScheduleTasksAction.java 2014-07-11 19:12:01 +0000
+++ dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/java/org/hisp/dhis/dataadmin/action/scheduling/ScheduleTasksAction.java 2014-07-11 21:22:57 +0000
@@ -33,11 +33,13 @@
import static org.hisp.dhis.scheduling.SchedulingManager.TASK_DATAMART_LAST_YEAR;
import static org.hisp.dhis.scheduling.SchedulingManager.TASK_RESOURCE_TABLE;
import static org.hisp.dhis.scheduling.SchedulingManager.TASK_MONITORING_LAST_DAY;
+import static org.hisp.dhis.scheduling.SchedulingManager.TASK_DATA_SYNCH;
import static org.hisp.dhis.setting.SystemSettingManager.DEFAULT_ORGUNITGROUPSET_AGG_LEVEL;
import static org.hisp.dhis.setting.SystemSettingManager.DEFAULT_SCHEDULED_PERIOD_TYPES;
import static org.hisp.dhis.setting.SystemSettingManager.KEY_ORGUNITGROUPSET_AGG_LEVEL;
import static org.hisp.dhis.setting.SystemSettingManager.KEY_SCHEDULED_PERIOD_TYPES;
import static org.hisp.dhis.system.scheduling.Scheduler.CRON_DAILY_0AM;
+import static org.hisp.dhis.system.scheduling.Scheduler.CRON_EVERY_MIN;
import static org.hisp.dhis.system.scheduling.Scheduler.STATUS_RUNNING;
import static org.hisp.dhis.system.util.CollectionUtils.emptyIfNull;
@@ -65,6 +67,7 @@
{
private static final String STRATEGY_ALL_DAILY = "allDaily";
private static final String STRATEGY_LAST_3_YEARS_DAILY = "last3YearsDaily";
+ private static final String STRATEGY_ENABLED = "enabled";
private static final Log log = LogFactory.getLog( ScheduleTasksAction.class );
@@ -176,6 +179,18 @@
this.monitoringStrategy = monitoringStrategy;
}
+ private String dataSynchStrategy;
+
+ public String getDataSynchStrategy()
+ {
+ return dataSynchStrategy;
+ }
+
+ public void setDataSynchStrategy( String dataSynchStrategy )
+ {
+ this.dataSynchStrategy = dataSynchStrategy;
+ }
+
// -------------------------------------------------------------------------
// Output
// -------------------------------------------------------------------------
@@ -260,13 +275,22 @@
{
cronKeyMap.putValue( CRON_DAILY_0AM, TASK_MONITORING_LAST_DAY );
}
+
+ // -------------------------------------------------------------
+ // Data synch
+ // -------------------------------------------------------------
+
+ if ( STRATEGY_ENABLED.equals( dataSynchStrategy ) )
+ {
+ cronKeyMap.putValue( CRON_EVERY_MIN, TASK_DATA_SYNCH );
+ }
schedulingManager.scheduleTasks( cronKeyMap );
}
}
else
{
- Collection<String> keys = emptyIfNull( schedulingManager.getCronKeyMap().get( CRON_DAILY_0AM ) );
+ Collection<String> keys = emptyIfNull( schedulingManager.getScheduledKeys() );
// -----------------------------------------------------------------
// Resource tables
@@ -307,6 +331,15 @@
{
monitoringStrategy = STRATEGY_ALL_DAILY;
}
+
+ // -------------------------------------------------------------
+ // Data synch
+ // -------------------------------------------------------------
+
+ if ( keys.contains( TASK_DATA_SYNCH ) )
+ {
+ dataSynchStrategy = STRATEGY_ENABLED;
+ }
}
scheduledPeriodTypes = (Set<String>) systemSettingManager.getSystemSetting( KEY_SCHEDULED_PERIOD_TYPES, DEFAULT_SCHEDULED_PERIOD_TYPES );
=== modified file 'dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/resources/org/hisp/dhis/dataadmin/i18n_module.properties'
--- dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/resources/org/hisp/dhis/dataadmin/i18n_module.properties 2014-07-11 19:38:35 +0000
+++ dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/resources/org/hisp/dhis/dataadmin/i18n_module.properties 2014-07-11 21:22:57 +0000
@@ -350,4 +350,7 @@
last_year_daily=Last year daily
show_fewer_options=Show fewer options
show_more_options=Show more options
-disabled=Disabled
\ No newline at end of file
+disabled=Disabled
+data_synchronization=Data synchronization
+data_synchronization_strategy=Data synchronization strategy
+enabled=Enabled
\ No newline at end of file
=== modified file 'dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/webapp/dhis-web-maintenance-dataadmin/javascript/scheduling.js'
--- dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/webapp/dhis-web-maintenance-dataadmin/javascript/scheduling.js 2014-07-11 19:12:01 +0000
+++ dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/webapp/dhis-web-maintenance-dataadmin/javascript/scheduling.js 2014-07-11 21:22:57 +0000
@@ -10,7 +10,7 @@
$( '.scheduling' ).removeAttr( 'disabled' );
}
- if ( $( '#dataMartStrategy' ).val() == 'never' )
+ if ( $( '#dataMartStrategy' ).val() == 'disabled' )
{
$( '.dataMart' ).attr( 'disabled', 'disabled' );
}
=== modified file 'dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/webapp/dhis-web-maintenance-dataadmin/viewScheduledTasks.vm'
--- dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/webapp/dhis-web-maintenance-dataadmin/viewScheduledTasks.vm 2014-07-11 19:38:35 +0000
+++ dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/webapp/dhis-web-maintenance-dataadmin/viewScheduledTasks.vm 2014-07-11 21:22:57 +0000
@@ -45,6 +45,19 @@
</select>
</div>
+<!-- Data value synch -->
+
+<div class="settingHeader">$i18n.getString( "data_synchronization" )</div>
+
+<div class="settingLabel settingIndent">$i18n.getString( "data_synchronization_strategy" )</div>
+
+<div class="setting settingIndent">
+<select id="dataSynchStrategy" name="dataSynchStrategy" class="scheduling">
+ <option value="disabled">$i18n.getString( "disabled" )</option>
+ <option value="enabled"#if( $dataSynchStrategy && $dataSynchStrategy == "enabled" ) selected="selected"#end>$i18n.getString( "enabled" )</option>
+</select>
+</div>
+
<div id="moreOptionsLink" style="margin-bottom:10px"><a href="javascript:toggleMoreOptions()">$i18n.getString( "show_more_options" )</a></div>
<div style="display:none" id="moreOptionsDiv">
