dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #14395
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 4884: Data mart: Filtering out leaf organisation units without data before export. Using organisation u...
Merge authors:
Lars Helge Øverland (larshelge)
------------------------------------------------------------
revno: 4884 [merge]
committer: Lars Helge Overland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Sat 2011-10-08 15:31:38 +0200
message:
Data mart: Filtering out leaf organisation units without data before export. Using organisation units for async prosess workload distribution, better option compared to periods when a larger number of cores is available.
modified:
dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitService.java
dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitStore.java
dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitService.java
dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/hibernate/HibernateOrganisationUnitStore.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java
dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/DhisConvenienceTest.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitService.java'
--- dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitService.java 2011-09-14 14:35:02 +0000
+++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitService.java 2011-10-08 13:09:04 +0000
@@ -269,6 +269,8 @@
OrganisationUnitDataSetAssociationSet getOrganisationUnitDataSetAssociationSet();
+ void filterOrganisationUnitsWithoutData( Collection<OrganisationUnit> organisationUnits );
+
// -------------------------------------------------------------------------
// OrganisationUnitHierarchy
// -------------------------------------------------------------------------
=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitStore.java'
--- dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitStore.java 2011-07-25 08:52:10 +0000
+++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitStore.java 2011-10-08 13:09:04 +0000
@@ -73,6 +73,10 @@
Collection<OrganisationUnit> getOrganisationUnitsWithoutGroups();
Collection<OrganisationUnit> getOrganisationUnitsByNameAndGroups( String name, Collection<OrganisationUnitGroup> groups, boolean limit );
+
+ Map<Integer, Set<Integer>> getOrganisationUnitDataSetAssocationMap();
+
+ Set<Integer> getOrganisationUnitIdsWithoutData();
// -------------------------------------------------------------------------
// OrganisationUnitHierarchy
@@ -93,8 +97,6 @@
*/
void updateOrganisationUnitParent( int organisationUnitId, int parentId );
- Map<Integer, Set<Integer>> getOrganisationUnitDataSetAssocationMap();
-
// -------------------------------------------------------------------------
// OrganisationUnitLevel
// -------------------------------------------------------------------------
=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitService.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitService.java 2011-09-14 14:35:02 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitService.java 2011-10-08 13:09:04 +0000
@@ -530,6 +530,19 @@
}
}
+ public void filterOrganisationUnitsWithoutData( Collection<OrganisationUnit> organisationUnits )
+ {
+ final Set<Integer> unitsWithoutData = organisationUnitStore.getOrganisationUnitIdsWithoutData();
+
+ FilterUtils.filter( organisationUnits, new Filter<OrganisationUnit>()
+ {
+ public boolean retain( OrganisationUnit unit )
+ {
+ return unit != null && ( !unitsWithoutData.contains( unit.getId() ) || unit.hasChild() );
+ }
+ } );
+ }
+
// -------------------------------------------------------------------------
// OrganisationUnitHierarchy
// -------------------------------------------------------------------------
=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/hibernate/HibernateOrganisationUnitStore.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/hibernate/HibernateOrganisationUnitStore.java 2011-07-25 08:52:10 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/hibernate/HibernateOrganisationUnitStore.java 2011-10-08 13:09:04 +0000
@@ -191,6 +191,24 @@
return map;
}
+ public Set<Integer> getOrganisationUnitIdsWithoutData()
+ {
+ final String sql = "select organisationunitid from organisationunit ou where not exists (" +
+ "select sourceid from datavalue where sourceid=ou.organisationunitid)";
+
+ final Set<Integer> units = new HashSet<Integer>();
+
+ jdbcTemplate.query( sql, new RowCallbackHandler()
+ {
+ public void processRow( ResultSet rs ) throws SQLException
+ {
+ units.add( rs.getInt( 1 ) );
+ }
+ } );
+
+ return units;
+ }
+
// -------------------------------------------------------------------------
// OrganisationUnitHierarchy
// -------------------------------------------------------------------------
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2011-09-24 12:01:19 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2011-10-08 13:31:38 +0000
@@ -29,6 +29,7 @@
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -58,8 +59,8 @@
import org.hisp.dhis.system.util.ConcurrentUtils;
import org.hisp.dhis.system.util.ConversionUtils;
import org.hisp.dhis.system.util.FilterUtils;
+import org.hisp.dhis.system.util.PaginatedList;
import org.hisp.dhis.system.util.SystemUtils;
-import org.hisp.dhis.system.util.WeightedPaginatedList;
import org.springframework.transaction.annotation.Transactional;
/**
@@ -167,9 +168,11 @@
Collection<Indicator> indicators = indicatorService.getIndicators( indicatorIds );
Collection<Period> periods = periodService.getPeriods( periodIds );
- Collection<OrganisationUnit> organisationUnits = organisationUnitService.getOrganisationUnits( organisationUnitIds );
+ List<OrganisationUnit> organisationUnits = new ArrayList<OrganisationUnit>( organisationUnitService.getOrganisationUnits( organisationUnitIds ) );
Collection<DataElement> dataElements = dataElementService.getDataElements( dataElementIds );
+ organisationUnitService.filterOrganisationUnitsWithoutData( organisationUnits );
+ Collections.shuffle( organisationUnits );
FilterUtils.filter( dataElements, new AggregatableDataElementFilter() );
expressionService.filterInvalidIndicators( indicators );
@@ -268,20 +271,20 @@
state.setMessage( "exporting_data_for_data_elements" );
- List<List<Period>> periodPages = new WeightedPaginatedList<Period>( periods, cpuCores ).getPages();
+ List<List<OrganisationUnit>> organisationUnitPages = new PaginatedList<OrganisationUnit>( organisationUnits ).setNumberOfPages( cpuCores ).getPages();
if ( allOperands.size() > 0 )
{
List<Future<?>> futures = new ArrayList<Future<?>>();
- for ( List<Period> periodPage : periodPages )
+ for ( List<OrganisationUnit> organisationUnitPage : organisationUnitPages )
{
- futures.add( dataElementDataMart.exportDataValues( allOperands, periodPage, organisationUnits, new DataElementOperandList( indicatorOperands ), key ) );
+ futures.add( dataElementDataMart.exportDataValues( allOperands, periods, organisationUnitPage, new DataElementOperandList( indicatorOperands ), key ) );
}
ConcurrentUtils.waitForCompletion( futures );
- clock.logTime( "Exported values for data element operands (" + allOperands.size() + "), number of pages: " + periodPages.size() );
+ clock.logTime( "Exported values for data element operands (" + allOperands.size() + "), number of pages: " + organisationUnitPages.size() );
}
// ---------------------------------------------------------------------
@@ -302,14 +305,14 @@
{
List<Future<?>> futures = new ArrayList<Future<?>>();
- for ( List<Period> periodPage : periodPages )
+ for ( List<OrganisationUnit> organisationUnitPage : organisationUnitPages )
{
- futures.add( indicatorDataMart.exportIndicatorValues( indicators, periodPage, organisationUnits, indicatorOperands, key ) );
+ futures.add( indicatorDataMart.exportIndicatorValues( indicators, periods, organisationUnitPage, indicatorOperands, key ) );
}
ConcurrentUtils.waitForCompletion( futures );
- clock.logTime( "Exported values for indicators (" + indicators.size() + "), number of pages: " + periodPages.size() );
+ clock.logTime( "Exported values for indicators (" + indicators.size() + "), number of pages: " + organisationUnitPages.size() );
}
// ---------------------------------------------------------------------
=== modified file 'dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/DhisConvenienceTest.java'
--- dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/DhisConvenienceTest.java 2011-09-24 11:35:11 +0000
+++ dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/DhisConvenienceTest.java 2011-10-08 13:09:04 +0000
@@ -590,6 +590,7 @@
OrganisationUnit unit = createOrganisationUnit( uniqueCharacter );
unit.setParent( parent );
+ parent.getChildren().add( unit );
return unit;
}
