dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #30559
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 15519: Std dev outlier analysis, paging org units in order to avoid very large sql criteria
------------------------------------------------------------
revno: 15519
committer: Lars Helge Overland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Mon 2014-06-02 15:44:57 +0200
message:
Std dev outlier analysis, paging org units in order to avoid very large sql criteria
modified:
dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataanalysis/jdbc/JdbcDataAnalysisStore.java
dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/Validator.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataanalysis/jdbc/JdbcDataAnalysisStore.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataanalysis/jdbc/JdbcDataAnalysisStore.java 2014-03-18 08:10:10 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataanalysis/jdbc/JdbcDataAnalysisStore.java 2014-06-02 13:44:57 +0000
@@ -36,9 +36,12 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.hisp.dhis.dataanalysis.DataAnalysisStore;
import org.hisp.dhis.dataelement.DataElement;
import org.hisp.dhis.dataelement.DataElementCategoryOptionCombo;
@@ -48,6 +51,7 @@
import org.hisp.dhis.period.Period;
import org.hisp.dhis.system.objectmapper.DeflatedDataValueNameMinMaxRowMapper;
import org.hisp.dhis.system.util.ConversionUtils;
+import org.hisp.dhis.system.util.PaginatedList;
import org.hisp.dhis.system.util.TextUtils;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.support.rowset.SqlRowSet;
@@ -58,6 +62,8 @@
public class JdbcDataAnalysisStore
implements DataAnalysisStore
{
+ private static final Log log = LogFactory.getLog( JdbcDataAnalysisStore.class );
+
// -------------------------------------------------------------------------
// Dependencies
// -------------------------------------------------------------------------
@@ -191,6 +197,25 @@
return new ArrayList<DeflatedDataValue>();
}
+ //TODO parallel processes
+
+ List<List<Integer>> organisationUnitPages = new PaginatedList<>( lowerBoundMap.keySet() ).setPageSize( 100 ).getPages();
+
+ log.debug( "No of pages: " + organisationUnitPages.size() );
+
+ Collection<DeflatedDataValue> dataValues = new ArrayList<>();
+
+ for ( List<Integer> unitPage : organisationUnitPages )
+ {
+ dataValues.addAll( getDeflatedDataValues( dataElement, categoryOptionCombo, periods, unitPage, lowerBoundMap, upperBoundMap ) );
+ }
+
+ return dataValues;
+ }
+
+ private Collection<DeflatedDataValue> getDeflatedDataValues( DataElement dataElement, DataElementCategoryOptionCombo categoryOptionCombo,
+ Collection<Period> periods, List<Integer> organisationUnits, Map<Integer, Integer> lowerBoundMap, Map<Integer, Integer> upperBoundMap )
+ {
String periodIds = TextUtils.getCommaDelimitedString( ConversionUtils.getIdentifiers( Period.class, periods ) );
String sql =
@@ -206,7 +231,7 @@
"and dv.categoryoptioncomboid = " + categoryOptionCombo.getId() + " " +
"and dv.periodid in (" + periodIds + ") and ( ";
- for ( Integer organisationUnit : lowerBoundMap.keySet() )
+ for ( Integer organisationUnit : organisationUnits )
{
sql += "( dv.sourceid = " + organisationUnit + " " +
"and ( cast( dv.value as " + statementBuilder.getDoubleColumnType() + " ) < " + lowerBoundMap.get( organisationUnit ) + " " +
=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/Validator.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/Validator.java 2014-05-27 02:41:16 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/Validator.java 2014-06-02 13:44:57 +0000
@@ -44,7 +44,6 @@
import org.hisp.dhis.period.PeriodService;
import org.hisp.dhis.system.util.SystemUtils;
import org.hisp.dhis.user.CurrentUserService;
-import org.hisp.dhis.user.UserCredentials;
import org.hisp.dhis.user.UserService;
/**