dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #15428
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 5581: Data mart, fixed lazy loading issue
------------------------------------------------------------
revno: 5581
committer: Lars Helge Overland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Sun 2011-12-25 12:16:00 +0100
message:
Data mart, fixed lazy loading issue
modified:
dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/jdbc/JdbcAggregatedDataValueStore.java
dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/jdbc/JdbcAggregatedOrgUnitDataValueStore.java
dhis-2/dhis-services/dhis-service-core/src/main/resources/META-INF/dhis/beans.xml
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DataElementDataMart.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DefaultDataElementDataMart.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/jdbc/JdbcAggregatedDataValueStore.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/jdbc/JdbcAggregatedDataValueStore.java 2011-12-25 10:13:51 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/jdbc/JdbcAggregatedDataValueStore.java 2011-12-25 11:16:00 +0000
@@ -86,7 +86,7 @@
this.statementManager = statementManager;
}
- protected JdbcTemplate jdbcTemplate;
+ private JdbcTemplate jdbcTemplate;
public void setJdbcTemplate( JdbcTemplate jdbcTemplate )
{
@@ -357,7 +357,7 @@
"AND periodid IN ( " + getCommaDelimitedString( periodIds ) + " ) " +
"AND organisationunitid IN ( " + getCommaDelimitedString( organisationUnitIds ) + " )";
- jdbcTemplate.execute( sql );
+ statementManager.getHolder().executeUpdate( sql );
}
public void deleteAggregatedDataValues( Collection<Integer> periodIds )
@@ -366,12 +366,14 @@
"DELETE FROM aggregateddatavalue " +
"WHERE periodid IN ( " + getCommaDelimitedString( periodIds ) + " )";
- jdbcTemplate.execute( sql );
+ statementManager.getHolder().executeUpdate( sql );
}
public void deleteAggregatedDataValues()
{
- jdbcTemplate.execute( "DELETE FROM aggregateddatavalue" );
+ final String sql = "DELETE FROM aggregateddatavalue";
+
+ statementManager.getHolder().executeUpdate( sql );
}
public void createIndex( boolean dataElement, boolean indicator )
@@ -558,7 +560,7 @@
"AND periodid IN ( " + getCommaDelimitedString( periodIds ) + " ) " +
"AND organisationunitid IN ( " + getCommaDelimitedString( organisationUnitIds ) + " )";
- jdbcTemplate.execute( sql );
+ statementManager.getHolder().executeUpdate( sql );
}
public void deleteAggregatedIndicatorValues( Collection<Integer> periodIds )
@@ -567,12 +569,14 @@
"DELETE FROM aggregatedindicatorvalue " +
"WHERE periodid IN ( " + getCommaDelimitedString( periodIds ) + " )";
- jdbcTemplate.execute( sql );
+ statementManager.getHolder().executeUpdate( sql );
}
public void deleteAggregatedIndicatorValues()
{
- jdbcTemplate.execute( "DELETE FROM aggregatedindicatorvalue" );
+ final String sql = "DELETE FROM aggregatedindicatorvalue";
+
+ statementManager.getHolder().executeUpdate( sql );
}
@Override
=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/jdbc/JdbcAggregatedOrgUnitDataValueStore.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/jdbc/JdbcAggregatedOrgUnitDataValueStore.java 2011-12-19 21:33:01 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/jdbc/JdbcAggregatedOrgUnitDataValueStore.java 2011-12-25 11:16:00 +0000
@@ -35,7 +35,6 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.hisp.dhis.aggregation.AggregatedOrgUnitDataValueStore;
-import org.springframework.jdbc.core.JdbcTemplate;
public class JdbcAggregatedOrgUnitDataValueStore
implements AggregatedOrgUnitDataValueStore
@@ -53,13 +52,6 @@
this.statementManager = statementManager;
}
- protected JdbcTemplate jdbcTemplate;
-
- public void setJdbcTemplate( JdbcTemplate jdbcTemplate )
- {
- this.jdbcTemplate = jdbcTemplate;
- }
-
// -------------------------------------------------------------------------
// AggregatedOrgUnitDataValueStore implementation
// -------------------------------------------------------------------------
@@ -86,7 +78,7 @@
"AND periodid IN ( " + getCommaDelimitedString( periodIds ) + " ) " +
"AND organisationunitid IN ( " + getCommaDelimitedString( organisationUnitIds ) + " )";
- jdbcTemplate.execute( sql );
+ statementManager.getHolder().executeUpdate( sql );
}
public void deleteAggregatedDataValues( Collection<Integer> periodIds )
@@ -95,7 +87,7 @@
"DELETE FROM aggregatedorgunitdatavalue " +
"WHERE periodid IN ( " + getCommaDelimitedString( periodIds ) + " )";
- jdbcTemplate.execute( sql );
+ statementManager.getHolder().executeUpdate( sql );
}
public void createIndex( boolean dataElement, boolean indicator )
@@ -105,7 +97,7 @@
try
{
final String sql = "CREATE INDEX aggregatedorgunitdatavalue_index ON aggregatedorgunitdatavalue (dataelementid, categoryoptioncomboid, periodid, organisationunitid, organisationunitgroupid)";
- jdbcTemplate.execute( sql );
+ statementManager.getHolder().executeUpdate( sql, true );
}
catch ( Exception ex )
{
@@ -118,7 +110,7 @@
try
{
final String sql = "CREATE INDEX aggregatedorgunitindicatorvalue_index ON aggregatedorgunitindicatorvalue (indicatorid, periodid, organisationunitid, organisationunitgroupid)";
- jdbcTemplate.execute( sql );
+ statementManager.getHolder().executeUpdate( sql, true );
}
catch ( Exception ex )
{
@@ -134,7 +126,7 @@
try
{
final String sql = "DROP INDEX aggregatedorgunitdatavalue_index";
- jdbcTemplate.execute( sql );
+ statementManager.getHolder().executeUpdate( sql, true );
}
catch ( Exception ex )
{
@@ -147,7 +139,7 @@
try
{
final String sql = "DROP INDEX aggregatedorgunitindicatorvalue_index";
- jdbcTemplate.execute( sql );
+ statementManager.getHolder().executeUpdate( sql, true );
}
catch ( Exception ex )
{
@@ -182,7 +174,7 @@
"AND periodid IN ( " + getCommaDelimitedString( periodIds ) + " ) " +
"AND organisationunitid IN ( " + getCommaDelimitedString( organisationUnitIds ) + " )";
- jdbcTemplate.execute( sql );
+ statementManager.getHolder().executeUpdate( sql );
}
public void deleteAggregatedIndicatorValues( Collection<Integer> periodIds )
@@ -191,6 +183,6 @@
"DELETE FROM aggregatedorgunitindicatorvalue " +
"WHERE periodid IN ( " + getCommaDelimitedString( periodIds ) + " )";
- jdbcTemplate.execute( sql );
+ statementManager.getHolder().executeUpdate( sql );
}
}
=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/resources/META-INF/dhis/beans.xml'
--- dhis-2/dhis-services/dhis-service-core/src/main/resources/META-INF/dhis/beans.xml 2011-12-11 13:47:04 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/resources/META-INF/dhis/beans.xml 2011-12-25 11:16:00 +0000
@@ -204,7 +204,6 @@
<bean id="org.hisp.dhis.aggregation.AggregatedOrgUnitDataValueStore" class="org.hisp.dhis.aggregation.jdbc.JdbcAggregatedOrgUnitDataValueStore">
<property name="statementManager" ref="statementManager" />
- <property name="jdbcTemplate" ref="jdbcTemplate" />
</bean>
<bean id="org.hisp.dhis.user.UserAuthorityGroupStore" class="org.hisp.dhis.hibernate.HibernateGenericStore">
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DataElementDataMart.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DataElementDataMart.java 2011-12-06 19:40:38 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DataElementDataMart.java 2011-12-25 11:16:00 +0000
@@ -36,6 +36,7 @@
import org.hisp.dhis.datamart.DataElementOperandList;
import org.hisp.dhis.organisationunit.OrganisationUnit;
import org.hisp.dhis.organisationunit.OrganisationUnitGroup;
+import org.hisp.dhis.organisationunit.OrganisationUnitHierarchy;
import org.hisp.dhis.period.Period;
/**
@@ -45,5 +46,5 @@
{
Future<?> exportDataValues( Collection<DataElementOperand> operands, Collection<Period> periods,
Collection<OrganisationUnit> organisationUnits, Collection<OrganisationUnitGroup> organisationUnitGroups,
- DataElementOperandList operandList, Class<? extends BatchHandler<AggregatedDataValue>> clazz, String key );
+ DataElementOperandList operandList, OrganisationUnitHierarchy hierarchy, Class<? extends BatchHandler<AggregatedDataValue>> clazz, String key );
}
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DefaultDataElementDataMart.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DefaultDataElementDataMart.java 2011-12-19 21:33:01 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DefaultDataElementDataMart.java 2011-12-25 11:16:00 +0000
@@ -52,7 +52,6 @@
import org.hisp.dhis.organisationunit.OrganisationUnit;
import org.hisp.dhis.organisationunit.OrganisationUnitGroup;
import org.hisp.dhis.organisationunit.OrganisationUnitHierarchy;
-import org.hisp.dhis.organisationunit.OrganisationUnitService;
import org.hisp.dhis.period.Period;
import org.springframework.scheduling.annotation.Async;
@@ -70,13 +69,6 @@
// Dependencies
// -------------------------------------------------------------------------
- private OrganisationUnitService organisationUnitService;
-
- public void setOrganisationUnitService( OrganisationUnitService organisationUnitService )
- {
- this.organisationUnitService = organisationUnitService;
- }
-
private BatchHandlerFactory batchHandlerFactory;
public void setBatchHandlerFactory( BatchHandlerFactory batchHandlerFactory )
@@ -147,7 +139,7 @@
@Async
public Future<?> exportDataValues( Collection<DataElementOperand> operands, Collection<Period> periods,
Collection<OrganisationUnit> organisationUnits, Collection<OrganisationUnitGroup> organisationUnitGroups,
- DataElementOperandList operandList, Class<? extends BatchHandler<AggregatedDataValue>> clazz, String key )
+ DataElementOperandList operandList, OrganisationUnitHierarchy hierarchy, Class<? extends BatchHandler<AggregatedDataValue>> clazz, String key )
{
statementManager.initialise(); // Running in separate thread
@@ -157,8 +149,6 @@
final BatchHandler<Object> cacheHandler = inMemoryBatchHandlerFactory.createBatchHandler( GenericBatchHandler.class ).setTableName( tableName + key ).init();
- final OrganisationUnitHierarchy hierarchy = organisationUnitService.getOrganisationUnitHierarchy().prepareChildren( organisationUnits, organisationUnitGroups );
-
final Map<DataElementOperand, Double> valueMap = new HashMap<DataElementOperand, Double>();
final AggregatedDataValue aggregatedValue = new AggregatedDataValue();
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2011-12-19 21:33:01 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2011-12-25 11:16:00 +0000
@@ -57,6 +57,7 @@
import org.hisp.dhis.organisationunit.OrganisationUnit;
import org.hisp.dhis.organisationunit.OrganisationUnitGroup;
import org.hisp.dhis.organisationunit.OrganisationUnitGroupService;
+import org.hisp.dhis.organisationunit.OrganisationUnitHierarchy;
import org.hisp.dhis.organisationunit.OrganisationUnitService;
import org.hisp.dhis.period.Period;
import org.hisp.dhis.period.PeriodService;
@@ -327,15 +328,17 @@
state.setMessage( "exporting_data_for_data_elements" );
List<List<OrganisationUnit>> organisationUnitPages = new PaginatedList<OrganisationUnit>( organisationUnits ).setNumberOfPages( cpuCores ).getPages();
-
+
if ( allOperands.size() > 0 )
{
+ final OrganisationUnitHierarchy hierarchy = organisationUnitService.getOrganisationUnitHierarchy().prepareChildren( organisationUnits );
+
List<Future<?>> futures = new ArrayList<Future<?>>();
for ( List<OrganisationUnit> organisationUnitPage : organisationUnitPages )
{
futures.add( dataElementDataMart.exportDataValues( allOperands, periods, organisationUnitPage,
- null, new DataElementOperandList( indicatorOperands ), AggregatedDataValueBatchHandler.class, key ) );
+ null, new DataElementOperandList( indicatorOperands ), hierarchy, AggregatedDataValueBatchHandler.class, key ) );
}
ConcurrentUtils.waitForCompletion( futures );
@@ -397,7 +400,9 @@
clock.logTime( "Created indexes" );
}
-
+
+ clock.logTime( "Aggregated data export done" );
+
final boolean isGroups = organisationUnitGroups != null && organisationUnitGroups.size() > 0;
final int groupLevel = (Integer) systemSettingManager.getSystemSetting( KEY_ORGUNITGROUPSET_AGG_LEVEL, DEFAULT_ORGUNITGROUPSET_AGG_LEVEL );
@@ -449,12 +454,14 @@
if ( allOperands.size() > 0 )
{
+ final OrganisationUnitHierarchy hierarchy = organisationUnitService.getOrganisationUnitHierarchy().prepareChildren( organisationUnits, organisationUnitGroups );
+
List<Future<?>> futures = new ArrayList<Future<?>>();
for ( List<OrganisationUnit> organisationUnitPage : organisationUnitPages )
{
futures.add( dataElementDataMart.exportDataValues( allOperands, periods, organisationUnitPage,
- organisationUnitGroups, new DataElementOperandList( indicatorOperands ), AggregatedOrgUnitDataValueBatchHandler.class, key ) );
+ organisationUnitGroups, new DataElementOperandList( indicatorOperands ), hierarchy, AggregatedOrgUnitDataValueBatchHandler.class, key ) );
}
ConcurrentUtils.waitForCompletion( futures );
@@ -516,6 +523,9 @@
clock.logTime( "Created org unit indexes" );
}
+
+ clock.logTime( "Aggregated organisation unit data export done" );
+
}
// ---------------------------------------------------------------------
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml 2011-12-19 16:44:56 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml 2011-12-25 11:16:00 +0000
@@ -95,7 +95,6 @@
<!-- DataElementDataMart -->
<bean id="org.hisp.dhis.datamart.dataelement.DataElementDataMart" class="org.hisp.dhis.datamart.dataelement.DefaultDataElementDataMart">
- <property name="organisationUnitService" ref="org.hisp.dhis.organisationunit.OrganisationUnitService" />
<property name="batchHandlerFactory" ref="batchHandlerFactory" />
<property name="inMemoryBatchHandlerFactory" ref="inMemoryBatchHandlerFactory" />
<property name="aggregationCache" ref="org.hisp.dhis.datamart.aggregation.cache.AggregationCache" />
