dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #14938
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 5214: Scheduling: made it possible to select one of two strategies for data mart scheduling, one where ...
------------------------------------------------------------
revno: 5214
committer: Lars Helge Overland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Thu 2011-11-24 17:04:56 +0100
message:
Scheduling: made it possible to select one of two strategies for data mart scheduling, one where data is generated for all months daily and another for last 6 months daily except sunday + from 6 to 12 months weekly on sunday. The last one should be sufficient for most usecases and will improve the capacity with large databases.
modified:
dhis-2/dhis-api/src/main/java/org/hisp/dhis/period/RelativePeriods.java
dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/scheduling/SchedulingManager.java
dhis-2/dhis-services/dhis-service-reporting/src/main/resources/META-INF/dhis/beans.xml
dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/DataMartTask.java
dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/Scheduler.java
dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/SpringScheduler.java
dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/java/org/hisp/dhis/dataadmin/action/scheduling/ScheduleTasksAction.java
dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/resources/org/hisp/dhis/dataadmin/i18n_module.properties
dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/webapp/dhis-web-maintenance-dataadmin/viewScheduledTasks.vm
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/period/RelativePeriods.java'
--- dhis-2/dhis-api/src/main/java/org/hisp/dhis/period/RelativePeriods.java 2011-11-14 12:19:36 +0000
+++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/period/RelativePeriods.java 2011-11-24 16:04:56 +0000
@@ -482,8 +482,8 @@
}
/**
- * Creates an instance of RelativePeriods based on the scheduledPeriodTypes
- * set from Configuration.
+ * Creates an instance of RelativePeriods based on given set of PeriodType
+ * names.
*
* @param config the Configuration object.
* @return a RelativePeriods instance.
=== modified file 'dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/scheduling/SchedulingManager.java'
--- dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/scheduling/SchedulingManager.java 2011-11-24 13:19:36 +0000
+++ dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/scheduling/SchedulingManager.java 2011-11-24 16:04:56 +0000
@@ -35,6 +35,8 @@
public interface SchedulingManager
{
final String TASK_DATAMART_LAST_12_MONTHS = "dataMartLast12MonthsTask";
+ final String TASK_DATAMART_LAST_6_MONTS = "dataMartLast6MonthsTask";
+ final String TASK_DATAMART_FROM_6_TO_12_MONTS = "dataMartFrom6To12MonthsTask";
void scheduleTasks();
=== modified file 'dhis-2/dhis-services/dhis-service-reporting/src/main/resources/META-INF/dhis/beans.xml'
--- dhis-2/dhis-services/dhis-service-reporting/src/main/resources/META-INF/dhis/beans.xml 2011-11-24 13:19:36 +0000
+++ dhis-2/dhis-services/dhis-service-reporting/src/main/resources/META-INF/dhis/beans.xml 2011-11-24 16:04:56 +0000
@@ -277,6 +277,8 @@
<property name="tasks">
<map>
<entry key="dataMartLast12MonthsTask" value-ref="dataMartLast12MonthsTask" />
+ <entry key="dataMartLast6MonthsTask" value-ref="dataMartLast6MonthsTask" />
+ <entry key="dataMartFrom6To12MonthsTask" value-ref="dataMartFrom6To12MonthsTask" />
</map>
</property>
</bean>
@@ -286,10 +288,19 @@
<constructor-arg ref="registrationDataCompletenessService" />
<constructor-arg ref="org.hisp.dhis.dataelement.DataElementService" />
<constructor-arg ref="org.hisp.dhis.indicator.IndicatorService" />
+ <constructor-arg ref="org.hisp.dhis.period.PeriodService" />
<constructor-arg ref="org.hisp.dhis.organisationunit.OrganisationUnitService" />
<constructor-arg ref="org.hisp.dhis.dataset.DataSetService" />
<constructor-arg ref="org.hisp.dhis.options.SystemSettingManager" />
</bean>
+
+ <bean id="dataMartLast6MonthsTask" class="org.hisp.dhis.system.scheduling.DataMartTask" parent="dataMartLast12MonthsTask">
+ <property name="last6Months" value="true"/>
+ </bean>
+
+ <bean id="dataMartFrom6To12MonthsTask" class="org.hisp.dhis.system.scheduling.DataMartTask" parent="dataMartLast12MonthsTask">
+ <property name="from6To12Months" value="true"/>
+ </bean>
<!-- DeletionHandler -->
=== modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/DataMartTask.java'
--- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/DataMartTask.java 2011-11-24 10:38:05 +0000
+++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/DataMartTask.java 2011-11-24 16:04:56 +0000
@@ -30,10 +30,14 @@
import static org.hisp.dhis.options.SystemSettingManager.DEFAULT_SCHEDULED_PERIOD_TYPES;
import static org.hisp.dhis.options.SystemSettingManager.KEY_SCHEDULED_PERIOD_TYPES;
+import java.util.Calendar;
import java.util.Collection;
-import java.util.HashSet;
+import java.util.Date;
+import java.util.List;
import java.util.Set;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.hisp.dhis.completeness.DataSetCompletenessService;
import org.hisp.dhis.dataelement.DataElement;
import org.hisp.dhis.dataelement.DataElementService;
@@ -45,8 +49,13 @@
import org.hisp.dhis.options.SystemSettingManager;
import org.hisp.dhis.organisationunit.OrganisationUnit;
import org.hisp.dhis.organisationunit.OrganisationUnitService;
+import org.hisp.dhis.period.Cal;
+import org.hisp.dhis.period.Period;
+import org.hisp.dhis.period.PeriodService;
import org.hisp.dhis.period.RelativePeriods;
import org.hisp.dhis.system.util.ConversionUtils;
+import org.hisp.dhis.system.util.Filter;
+import org.hisp.dhis.system.util.FilterUtils;
/**
* @author Lars Helge Overland
@@ -54,6 +63,8 @@
public class DataMartTask
implements Runnable
{
+ private static final Log log = LogFactory.getLog( DataMartTask.class );
+
private DataMartService dataMartService;
private DataSetCompletenessService completenessService;
@@ -61,21 +72,38 @@
private DataElementService dataElementService;
private IndicatorService indicatorService;
+
+ private PeriodService periodService;
private OrganisationUnitService organisationUnitService;
private DataSetService dataSetService;
private SystemSettingManager systemSettingManager;
+
+ private boolean last6Months;
+
+ public void setLast6Months( boolean last6Months )
+ {
+ this.last6Months = last6Months;
+ }
+
+ private boolean from6To12Months;
+ public void setFrom6To12Months( boolean from6To12Months )
+ {
+ this.from6To12Months = from6To12Months;
+ }
+
public DataMartTask( DataMartService dataMartService, DataSetCompletenessService completenessService,
- DataElementService dataElementService, IndicatorService indicatorService,
+ DataElementService dataElementService, IndicatorService indicatorService, PeriodService periodService,
OrganisationUnitService organisationUnitService, DataSetService dataSetService, SystemSettingManager systemSettingManager )
{
this.dataMartService = dataMartService;
this.completenessService = completenessService;
this.dataElementService = dataElementService;
this.indicatorService = indicatorService;
+ this.periodService = periodService;
this.organisationUnitService = organisationUnitService;
this.dataSetService = dataSetService;
this.systemSettingManager = systemSettingManager;
@@ -91,10 +119,37 @@
Collection<Integer> dataSetIds = ConversionUtils.getIdentifiers( DataSet.class, dataSetService.getAllDataSets() );
Set<String> periodTypes = (Set<String>) systemSettingManager.getSystemSetting( KEY_SCHEDULED_PERIOD_TYPES, DEFAULT_SCHEDULED_PERIOD_TYPES );
-
- RelativePeriods relatives = new RelativePeriods().getRelativePeriods( periodTypes );
-
- dataMartService.export( dataElementIds, indicatorIds, new HashSet<Integer>(), organisationUnitIds, relatives, true );
- completenessService.exportDataSetCompleteness( dataSetIds, relatives, organisationUnitIds );
+
+ List<Period> periods = new RelativePeriods().getRelativePeriods( periodTypes ).getRelativePeriods();
+
+ final Date date = new Cal().now().subtract( Calendar.MONTH, 6 ).time();
+
+ if ( last6Months )
+ {
+ FilterUtils.filter( periods, new Filter<Period>()
+ {
+ public boolean retain( Period period )
+ {
+ return period != null && period.getStartDate().compareTo( date ) > 0;
+ }
+ } );
+ }
+ else if ( from6To12Months )
+ {
+ FilterUtils.filter( periods, new Filter<Period>()
+ {
+ public boolean retain( Period period )
+ {
+ return period != null && period.getStartDate().compareTo( date ) <= 0;
+ }
+ } );
+ }
+
+ log.info( "Using periods: " + periods );
+
+ Collection<Integer> periodIds = ConversionUtils.getIdentifiers( Period.class, periodService.reloadPeriods( periods ) );
+
+ dataMartService.export( dataElementIds, indicatorIds, periodIds, organisationUnitIds, null, true );
+ completenessService.exportDataSetCompleteness( dataSetIds, periodIds, organisationUnitIds );
}
}
=== modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/Scheduler.java'
--- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/Scheduler.java 2011-11-24 13:19:36 +0000
+++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/Scheduler.java 2011-11-24 16:04:56 +0000
@@ -32,8 +32,10 @@
*/
public interface Scheduler
{
- final String CRON_NIGHTLY_0AM = "0 0 0 * * ?";
- final String CRON_NIGHTLY_1AM = "0 0 1 * * ?";
+ final String CRON_DAILY_0AM = "0 0 0 * * ?";
+ final String CRON_DAILY_1AM = "0 0 1 * * ?";
+ final String CRON_DAILY_0AM_EXCEPT_SUNDAY = "0 0 0 ? * 2-7";
+ final String CRON_WEEKLY_SUNDAY_0AM = "0 0 0 ? * 1";
final String CRON_TEST = "0 * * * * ?";
final String STATUS_RUNNING = "running";
=== modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/SpringScheduler.java'
--- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/SpringScheduler.java 2011-11-24 13:19:36 +0000
+++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/SpringScheduler.java 2011-11-24 16:04:56 +0000
@@ -28,6 +28,7 @@
*/
import java.util.HashMap;
+import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ScheduledFuture;
@@ -82,7 +83,7 @@
futures.put( key, future );
- log.info( "Scheduled task with key: " + key );
+ log.info( "Scheduled task with key: " + key + " and cron: " + cronExpr );
return true;
}
@@ -100,7 +101,7 @@
futures.remove( key );
- log.info( "Stopped task with key: " + key );
+ log.info( "Stopped task with key: " + key + " successfully: " + result );
return result;
}
@@ -110,9 +111,19 @@
public void stopAllTasks()
{
- for ( String key : futures.keySet() )
+ Iterator<String> keys = futures.keySet().iterator();
+
+ while ( keys.hasNext() )
{
- stopTask( key );
+ String key = keys.next();
+
+ ScheduledFuture<?> future = futures.get( key );
+
+ boolean result = future != null ? future.cancel( true ) : false;
+
+ keys.remove();
+
+ log.info( "Stopped task with key: " + key + " successfully: " + result );
}
}
=== modified file 'dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/java/org/hisp/dhis/dataadmin/action/scheduling/ScheduleTasksAction.java'
--- dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/java/org/hisp/dhis/dataadmin/action/scheduling/ScheduleTasksAction.java 2011-11-24 13:19:36 +0000
+++ dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/java/org/hisp/dhis/dataadmin/action/scheduling/ScheduleTasksAction.java 2011-11-24 16:04:56 +0000
@@ -47,8 +47,9 @@
public class ScheduleTasksAction
implements Action
{
- private static final String STRATEGY_DATAMART_LAST_12_MONTHS_DAILY = "dataMartLast12MonthsDaily";
-
+ private static final String STRATEGY_LAST_12_DAILY = "last12Daily";
+ private static final String STRATEGY_LAST_6_DAILY_6_TO_12_WEEKLY = "last6Daily6To12Weekly";
+
// -------------------------------------------------------------------------
// Dependencies
// -------------------------------------------------------------------------
@@ -94,6 +95,11 @@
private String dataMartStrategy;
+ public String getDataMartStrategy()
+ {
+ return dataMartStrategy;
+ }
+
public void setDataMartStrategy( String dataMartStrategy )
{
this.dataMartStrategy = dataMartStrategy;
@@ -147,14 +153,24 @@
{
Map<String, String> keyCronMap = new HashMap<String, String>();
- if ( STRATEGY_DATAMART_LAST_12_MONTHS_DAILY.equals( dataMartStrategy ) )
- {
- keyCronMap.put( SchedulingManager.TASK_DATAMART_LAST_12_MONTHS, Scheduler.CRON_NIGHTLY_0AM );
+ if ( STRATEGY_LAST_12_DAILY.equals( dataMartStrategy ) )
+ {
+ keyCronMap.put( SchedulingManager.TASK_DATAMART_LAST_12_MONTHS, Scheduler.CRON_DAILY_0AM );
+ }
+ else if ( STRATEGY_LAST_6_DAILY_6_TO_12_WEEKLY.equals( dataMartStrategy ) )
+ {
+ keyCronMap.put( SchedulingManager.TASK_DATAMART_LAST_6_MONTS, Scheduler.CRON_DAILY_0AM_EXCEPT_SUNDAY );
+ keyCronMap.put( SchedulingManager.TASK_DATAMART_FROM_6_TO_12_MONTS, Scheduler.CRON_WEEKLY_SUNDAY_0AM );
}
schedulingManager.scheduleTasks( keyCronMap );
}
}
+ else
+ {
+ dataMartStrategy = schedulingManager.getScheduledTasks().containsKey( SchedulingManager.TASK_DATAMART_LAST_12_MONTHS ) ?
+ STRATEGY_LAST_12_DAILY : STRATEGY_LAST_6_DAILY_6_TO_12_WEEKLY;
+ }
status = schedulingManager.getTaskStatus();
running = Scheduler.STATUS_RUNNING.equals( status );
=== modified file 'dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/resources/org/hisp/dhis/dataadmin/i18n_module.properties'
--- dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/resources/org/hisp/dhis/dataadmin/i18n_module.properties 2011-11-02 14:39:42 +0000
+++ dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/resources/org/hisp/dhis/dataadmin/i18n_module.properties 2011-11-24 16:04:56 +0000
@@ -369,4 +369,8 @@
not_started = not started
running = running
done = done
-stopped = stopped
\ No newline at end of file
+stopped = stopped
+datamart_task_strategy = Data mart task strategy
+never = Never
+last_12_months_daily = Last 12 months daily
+last_6_months_daily_6_to_12_months_weekly = Last 6 months daily + 6 to 12 months weekly
\ No newline at end of file
=== modified file 'dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/webapp/dhis-web-maintenance-dataadmin/viewScheduledTasks.vm'
--- dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/webapp/dhis-web-maintenance-dataadmin/viewScheduledTasks.vm 2011-11-24 13:19:36 +0000
+++ dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/webapp/dhis-web-maintenance-dataadmin/viewScheduledTasks.vm 2011-11-24 16:04:56 +0000
@@ -5,8 +5,6 @@
<form id="schedulingForm" action="scheduleTasks.action" method="post">
-<input id="dataMartStrategy" name="dataMartStrategy" type="hidden" value="dataMartLast12MonthsDaily">
-
<table>
<tr>
<th>$i18n.getString( "aggregation_period_types" )</th>
@@ -22,6 +20,21 @@
</tr>
<tr>
<td style="height:10px"></td>
+</tr>
+<tr>
+ <th>$i18n.getString( "datamart_task_strategy" )</th>
+</tr>
+<tr>
+ <td>
+ <select id="dataMartStrategy" name="dataMartStrategy" class="scheduling">
+ <option value="never">$i18n.getString( "never" )</option>
+ <option value="last12Daily"#if( $dataMartStrategy && $dataMartStrategy == "last12Daily" ) selected="selected"#end>$i18n.getString( "last_12_months_daily" )</option>
+ <option value="last6Daily6To12Weekly"#if( $dataMartStrategy && $dataMartStrategy == "last6Daily6To12Weekly" ) selected="selected"#end>$i18n.getString( "last_6_months_daily_6_to_12_months_weekly" )</option>
+ </select>
+ </td>
+</tr>
+<tr>
+ <td style="height:10px"></td>
</tr>
<tr>
<td>
