dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #23107
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 11229: Maintenance, current approach for clearing data mart by deleting all rows is slow, changed to dro...
------------------------------------------------------------
revno: 11229
committer: Lars Helge Øverland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Wed 2013-06-19 15:43:51 +0200
message:
Maintenance, current approach for clearing data mart by deleting all rows is slow, changed to dropping and recreating the tables instead.
modified:
dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedDataValueService.java
dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedDataValueStore.java
dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/DefaultAggregatedDataValueService.java
dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/jdbc/JdbcAggregatedDataValueStore.java
dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/startup/TableCreator.java
dhis-2/dhis-services/dhis-service-core/src/main/resources/META-INF/dhis/beans.xml
dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/java/org/hisp/dhis/dataadmin/action/maintenance/PerformMaintenanceAction.java
dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/resources/META-INF/dhis/beans.xml
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedDataValueService.java'
--- dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedDataValueService.java 2013-06-08 21:51:15 +0000
+++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedDataValueService.java 2013-06-19 13:43:51 +0000
@@ -283,4 +283,18 @@
*/
Collection<DataSetCompletenessResult> getAggregatedDataSetCompleteness( Collection<Integer> dataSetIds, Collection<Integer> periodIds,
Collection<Integer> organisationUnitIds );
+
+ // ----------------------------------------------------------------------
+ // Data mart
+ // ----------------------------------------------------------------------
+
+ /**
+ * Drops all data mart tables.
+ */
+ void dropDataMart();
+
+ /**
+ * Creates all data mart tables.
+ */
+ void createDataMart();
}
=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedDataValueStore.java'
--- dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedDataValueStore.java 2013-06-08 21:51:15 +0000
+++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedDataValueStore.java 2013-06-19 13:43:51 +0000
@@ -261,4 +261,18 @@
*/
Collection<DataSetCompletenessResult> getAggregatedDataSetCompleteness( Collection<Integer> dataSetIds, Collection<Integer> periodIds,
Collection<Integer> organisationUnitIds );
+
+ // ----------------------------------------------------------------------
+ // Data mart
+ // ----------------------------------------------------------------------
+
+ /**
+ * Drops all data mart tables.
+ */
+ void dropDataMart();
+
+ /**
+ * Creates all data mart tables.
+ */
+ void createDataMart();
}
=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/DefaultAggregatedDataValueService.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/DefaultAggregatedDataValueService.java 2013-06-08 21:51:15 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/DefaultAggregatedDataValueService.java 2013-06-19 13:43:51 +0000
@@ -189,4 +189,18 @@
{
return aggregatedDataValueStore.getAggregatedDataSetCompleteness( dataSetIds, periodIds, organisationUnitIds );
}
+
+ // -------------------------------------------------------------------------
+ // Data mart
+ // -------------------------------------------------------------------------
+
+ public void dropDataMart()
+ {
+ aggregatedDataValueStore.dropDataMart();
+ }
+
+ public void createDataMart()
+ {
+ aggregatedDataValueStore.createDataMart();
+ }
}
=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/jdbc/JdbcAggregatedDataValueStore.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/jdbc/JdbcAggregatedDataValueStore.java 2013-06-08 21:51:15 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/jdbc/JdbcAggregatedDataValueStore.java 2013-06-19 13:43:51 +0000
@@ -46,6 +46,7 @@
import org.hisp.dhis.dataelement.DataElement;
import org.hisp.dhis.dataelement.DataElementCategoryOption;
import org.hisp.dhis.dataelement.DataElementCategoryOptionCombo;
+import org.hisp.dhis.jdbc.StatementBuilder;
import org.hisp.dhis.organisationunit.OrganisationUnit;
import org.hisp.dhis.organisationunit.OrganisationUnitLevel;
import org.hisp.dhis.period.Period;
@@ -72,6 +73,13 @@
{
this.jdbcTemplate = jdbcTemplate;
}
+
+ private StatementBuilder statementBuilder;
+
+ public void setStatementBuilder( StatementBuilder statementBuilder )
+ {
+ this.statementBuilder = statementBuilder;
+ }
private StatementManager statementManager; //TODO remove
@@ -401,4 +409,40 @@
return jdbcTemplate.query( sql, new AggregatedDataSetCompletenessRowMapper() );
}
+
+ public void dropDataMart()
+ {
+ executeSilently( "drop table aggregateddatavalue" );
+ executeSilently( "drop table aggregatedorgunitdatavalue" );
+ executeSilently( "drop table aggregatedindicatorvalue" );
+ executeSilently( "drop table aggregatedorgunitindicatorvalue" );
+ executeSilently( "drop table aggregateddatasetcompleteness" );
+ executeSilently( "drop table aggregatedorgunitdatasetcompleteness" );
+ }
+
+ public void createDataMart()
+ {
+ executeSilently( statementBuilder.getCreateAggregatedDataValueTable( false ) );
+ executeSilently( statementBuilder.getCreateAggregatedOrgUnitDataValueTable( false ) );
+ executeSilently( statementBuilder.getCreateAggregatedIndicatorTable( false ) );
+ executeSilently( statementBuilder.getCreateAggregatedOrgUnitIndicatorTable( false ) );
+ executeSilently( statementBuilder.getCreateDataSetCompletenessTable() );
+ executeSilently( statementBuilder.getCreateOrgUnitDataSetCompletenessTable() );
+ }
+
+ // -------------------------------------------------------------------------
+ // Supportive
+ // -------------------------------------------------------------------------
+
+ private void executeSilently( final String sql )
+ {
+ try
+ {
+ jdbcTemplate.execute( sql );
+ }
+ catch ( Exception ex )
+ {
+ // Ignore
+ }
+ }
}
=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/startup/TableCreator.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/startup/TableCreator.java 2013-04-30 08:03:51 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/startup/TableCreator.java 2013-06-19 13:43:51 +0000
@@ -29,8 +29,9 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.hisp.dhis.jdbc.StatementBuilder;
+import org.hisp.dhis.aggregation.AggregatedDataValueService;
import org.hisp.dhis.system.startup.AbstractStartupRoutine;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
/**
@@ -45,19 +46,11 @@
// Dependencies
// -------------------------------------------------------------------------
+ @Autowired
private JdbcTemplate jdbcTemplate;
-
- public void setJdbcTemplate( JdbcTemplate jdbcTemplate )
- {
- this.jdbcTemplate = jdbcTemplate;
- }
-
- private StatementBuilder statementBuilder;
-
- public void setStatementBuilder( StatementBuilder statementBuilder )
- {
- this.statementBuilder = statementBuilder;
- }
+
+ @Autowired
+ private AggregatedDataValueService aggregatedDataValueService;
// -------------------------------------------------------------------------
// StartupRoutine implementation
@@ -65,12 +58,7 @@
public void execute()
{
- createSilently( statementBuilder.getCreateAggregatedDataValueTable( false ), "aggregateddatavalue" );
- createSilently( statementBuilder.getCreateAggregatedOrgUnitDataValueTable( false ), "aggregatedorgunitdatavalue" );
- createSilently( statementBuilder.getCreateAggregatedIndicatorTable( false ), "aggregatedindicatorvalue" );
- createSilently( statementBuilder.getCreateAggregatedOrgUnitIndicatorTable( false ), "aggregatedorgunitindicatorvalue" );
- createSilently( statementBuilder.getCreateDataSetCompletenessTable(), "aggregateddatasetcompleteness" );
- createSilently( statementBuilder.getCreateOrgUnitDataSetCompletenessTable(), "aggregatedorgunitdatasetcompleteness" );
+ aggregatedDataValueService.createDataMart();
createSilently( "CREATE INDEX crosstab ON datavalue ( periodid, sourceid )", "crosstab" );
createSilently( "CREATE INDEX messageconversation_lastmessage ON messageconversation (lastmessage)", "messageconversation_lastmessage" );
=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/resources/META-INF/dhis/beans.xml'
--- dhis-2/dhis-services/dhis-service-core/src/main/resources/META-INF/dhis/beans.xml 2013-06-10 11:42:40 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/resources/META-INF/dhis/beans.xml 2013-06-19 13:43:51 +0000
@@ -235,6 +235,7 @@
<bean id="org.hisp.dhis.aggregation.AggregatedDataValueStore" class="org.hisp.dhis.aggregation.jdbc.JdbcAggregatedDataValueStore">
<property name="statementManager" ref="statementManager" />
+ <property name="statementBuilder" ref="statementBuilder" />
<property name="jdbcTemplate" ref="jdbcTemplate" />
</bean>
@@ -762,8 +763,6 @@
</bean>
<bean id="org.hisp.dhis.startup.TableCreator" class="org.hisp.dhis.startup.TableCreator">
- <property name="jdbcTemplate" ref="jdbcTemplate" />
- <property name="statementBuilder" ref="statementBuilder" />
<property name="name" value="TableCreator" />
<property name="runlevel" value="3" />
</bean>
=== modified file 'dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/java/org/hisp/dhis/dataadmin/action/maintenance/PerformMaintenanceAction.java'
--- dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/java/org/hisp/dhis/dataadmin/action/maintenance/PerformMaintenanceAction.java 2013-04-22 08:10:07 +0000
+++ dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/java/org/hisp/dhis/dataadmin/action/maintenance/PerformMaintenanceAction.java 2013-06-19 13:43:51 +0000
@@ -32,7 +32,6 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.hisp.dhis.aggregation.AggregatedDataValueService;
-import org.hisp.dhis.aggregation.AggregatedOrgUnitDataValueService;
import org.hisp.dhis.analytics.AnalyticsTableService;
import org.hisp.dhis.common.DeleteNotAllowedException;
import org.hisp.dhis.completeness.DataSetCompletenessService;
@@ -86,14 +85,7 @@
{
this.aggregatedDataValueService = aggregatedDataValueService;
}
-
- private AggregatedOrgUnitDataValueService aggregatedOrgUnitDataValueService;
-
- public void setAggregatedOrgUnitDataValueService( AggregatedOrgUnitDataValueService aggregatedOrgUnitDataValueService )
- {
- this.aggregatedOrgUnitDataValueService = aggregatedOrgUnitDataValueService;
- }
-
+
private DataMartManager dataMartManager;
public void setDataMartManager( DataMartManager dataMartManager )
@@ -177,11 +169,8 @@
if ( clearDataMart )
{
- aggregatedDataValueService.deleteAggregatedDataValues();
- aggregatedDataValueService.deleteAggregatedIndicatorValues();
-
- aggregatedOrgUnitDataValueService.deleteAggregatedDataValues();
- aggregatedOrgUnitDataValueService.deleteAggregatedIndicatorValues();
+ aggregatedDataValueService.dropDataMart();
+ aggregatedDataValueService.createDataMart();
log.info( "'" + currentUserService.getCurrentUsername() + "': Cleared data mart" );
}
=== modified file 'dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/resources/META-INF/dhis/beans.xml'
--- dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/resources/META-INF/dhis/beans.xml 2013-04-30 08:03:51 +0000
+++ dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/resources/META-INF/dhis/beans.xml 2013-06-19 13:43:51 +0000
@@ -11,7 +11,6 @@
<property name="maintenanceService" ref="org.hisp.dhis.maintenance.MaintenanceService" />
<property name="completenessService" ref="compulsoryDataCompletenessService" />
<property name="aggregatedDataValueService" ref="org.hisp.dhis.aggregation.AggregatedDataValueService" />
- <property name="aggregatedOrgUnitDataValueService" ref="org.hisp.dhis.aggregation.AggregatedOrgUnitDataValueService" />
<property name="dataMartManager" ref="org.hisp.dhis.datamart.DataMartManager" />
<property name="periodService" ref="org.hisp.dhis.period.PeriodService" />
<property name="currentUserService" ref="org.hisp.dhis.user.CurrentUserService" />
