dhis2-devs team mailing list archive

[noreply@xxxxxxxxxxxxx]

------------------------------------------------------------
revno: 15519
committer: Lars Helge Overland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Mon 2014-06-02 15:44:57 +0200
message:
  Std dev outlier analysis, paging org units in order to avoid very large sql criteria
modified:
  dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataanalysis/jdbc/JdbcDataAnalysisStore.java
  dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/Validator.java


--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk

Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription

=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataanalysis/jdbc/JdbcDataAnalysisStore.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataanalysis/jdbc/JdbcDataAnalysisStore.java	2014-03-18 08:10:10 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataanalysis/jdbc/JdbcDataAnalysisStore.java	2014-06-02 13:44:57 +0000
@@ -36,9 +36,12 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.hisp.dhis.dataanalysis.DataAnalysisStore;
 import org.hisp.dhis.dataelement.DataElement;
 import org.hisp.dhis.dataelement.DataElementCategoryOptionCombo;
@@ -48,6 +51,7 @@
 import org.hisp.dhis.period.Period;
 import org.hisp.dhis.system.objectmapper.DeflatedDataValueNameMinMaxRowMapper;
 import org.hisp.dhis.system.util.ConversionUtils;
+import org.hisp.dhis.system.util.PaginatedList;
 import org.hisp.dhis.system.util.TextUtils;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.jdbc.support.rowset.SqlRowSet;
@@ -58,6 +62,8 @@
 public class JdbcDataAnalysisStore
     implements DataAnalysisStore
 {
+    private static final Log log = LogFactory.getLog( JdbcDataAnalysisStore.class );
+    
     // -------------------------------------------------------------------------
     // Dependencies
     // -------------------------------------------------------------------------
@@ -191,6 +197,25 @@
             return new ArrayList<DeflatedDataValue>();
         }
         
+        //TODO parallel processes
+                
+        List<List<Integer>> organisationUnitPages = new PaginatedList<>( lowerBoundMap.keySet() ).setPageSize( 100 ).getPages();
+        
+        log.debug( "No of pages: " + organisationUnitPages.size() );
+        
+        Collection<DeflatedDataValue> dataValues = new ArrayList<>();
+        
+        for ( List<Integer> unitPage : organisationUnitPages )
+        {
+            dataValues.addAll( getDeflatedDataValues( dataElement, categoryOptionCombo, periods, unitPage, lowerBoundMap, upperBoundMap ) );
+        }
+        
+        return dataValues;
+    }
+    
+    private Collection<DeflatedDataValue> getDeflatedDataValues( DataElement dataElement, DataElementCategoryOptionCombo categoryOptionCombo,
+        Collection<Period> periods, List<Integer> organisationUnits, Map<Integer, Integer> lowerBoundMap, Map<Integer, Integer> upperBoundMap )
+    {
         String periodIds = TextUtils.getCommaDelimitedString( ConversionUtils.getIdentifiers( Period.class, periods ) );
         
         String sql = 
@@ -206,7 +231,7 @@
             "and dv.categoryoptioncomboid = " + categoryOptionCombo.getId() + " " +
             "and dv.periodid in (" + periodIds + ") and ( ";
         
-        for ( Integer organisationUnit : lowerBoundMap.keySet() )
+        for ( Integer organisationUnit : organisationUnits )
         {
             sql += "( dv.sourceid = " + organisationUnit + " " +
                 "and ( cast( dv.value as " + statementBuilder.getDoubleColumnType() + " ) < " + lowerBoundMap.get( organisationUnit ) + " " +

=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/Validator.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/Validator.java	2014-05-27 02:41:16 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/Validator.java	2014-06-02 13:44:57 +0000
@@ -44,7 +44,6 @@
 import org.hisp.dhis.period.PeriodService;
 import org.hisp.dhis.system.util.SystemUtils;
 import org.hisp.dhis.user.CurrentUserService;
-import org.hisp.dhis.user.UserCredentials;
 import org.hisp.dhis.user.UserService;
 
 /**