dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #35840
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 18356: Impl support for cache preheat option in data value import, so far for org unit. Added option in ...
------------------------------------------------------------
revno: 18356
committer: Lars Helge Overland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Fri 2015-02-20 12:17:31 +0100
message:
Impl support for cache preheat option in data value import, so far for org unit. Added option in data value import screen. Impl CachingMap. Could have used guava cache but it has awkward null handling, it is also convenient to work with the standard Map interface for method-scoped caches.
added:
dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/CachingMap.java
modified:
dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/common/ImportOptions.java
dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DefaultDataValueSetService.java
dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/datavalue/ImportDataValueAction.java
dhis-2/dhis-web/dhis-web-importexport/src/main/resources/org/hisp/dhis/importexport/i18n_module.properties
dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/importDataValue.vm
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/common/ImportOptions.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/common/ImportOptions.java 2015-02-17 06:00:52 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/common/ImportOptions.java 2015-02-20 11:17:31 +0000
@@ -221,6 +221,7 @@
public String toString()
{
return "[Id scheme: " + idScheme + ", data element id scheme: " + dataElementIdScheme + ", org unit id scheme: " +
- orgUnitIdScheme + ", dry run: " + dryRun + ", async: " + async + ", strategy: " + importStrategy + ", skip check: " + skipExistingCheck + "]";
+ orgUnitIdScheme + ", dry run: " + dryRun + ", preheat cache: " + preheatCache + ", async: " +
+ async + ", strategy: " + importStrategy + ", skip check: " + skipExistingCheck + "]";
}
}
=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DefaultDataValueSetService.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DefaultDataValueSetService.java 2015-02-19 22:12:40 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DefaultDataValueSetService.java 2015-02-20 11:17:31 +0000
@@ -87,7 +87,9 @@
import org.hisp.dhis.period.PeriodService;
import org.hisp.dhis.period.PeriodType;
import org.hisp.dhis.scheduling.TaskId;
+import org.hisp.dhis.system.callable.IdentifiableObjectCallable;
import org.hisp.dhis.system.notification.Notifier;
+import org.hisp.dhis.system.util.CachingMap;
import org.hisp.dhis.system.util.DateUtils;
import org.hisp.dhis.system.util.DebugUtils;
import org.hisp.dhis.system.util.ValidationUtils;
@@ -620,15 +622,25 @@
boolean skipExistingCheck = importOptions.isSkipExistingCheck();
//----------------------------------------------------------------------
- // Build meta-data maps
+ // Create meta-data maps
//----------------------------------------------------------------------
Map<String, DataElement> dataElementMap = identifiableObjectManager.getIdMap( DataElement.class, dataElementIdScheme );
- Map<String, OrganisationUnit> orgUnitMap = getOrgUnitMap( orgUnitIdScheme );
+ CachingMap<String, OrganisationUnit> orgUnitMap = new CachingMap<String, OrganisationUnit>();
Map<String, DataElementCategoryOptionCombo> categoryOptionComboMap = identifiableObjectManager.getIdMap( DataElementCategoryOptionCombo.class, idScheme );
Map<String, Period> periodMap = new HashMap<>();
//----------------------------------------------------------------------
+ // Load meta-data maps
+ //----------------------------------------------------------------------
+
+ if ( importOptions.isPreheatCache() )
+ {
+ notifier.notify( id, "Loading organisation units" ).addTaskSummary( id, summary );
+ orgUnitMap.putAll( getOrgUnitMap( orgUnitIdScheme ) );
+ }
+
+ //----------------------------------------------------------------------
// Get outer meta-data
//----------------------------------------------------------------------
@@ -638,7 +650,8 @@
Period outerPeriod = PeriodType.getPeriodFromIsoString( trimToNull( dataValueSet.getPeriod() ) );
- OrganisationUnit outerOrgUnit = dataValueSet.getOrgUnit() != null ? orgUnitMap.get( dataValueSet.getOrgUnit() ) : null;
+ OrganisationUnit outerOrgUnit = orgUnitMap.get( trimToNull( dataValueSet.getOrgUnit() ),
+ new IdentifiableObjectCallable<>( identifiableObjectManager, OrganisationUnit.class, trimToNull( dataValueSet.getOrgUnit() ) ) );
DataElementCategoryOptionCombo fallbackCategoryOptionCombo = categoryService.getDefaultDataElementCategoryOptionCombo();
@@ -712,7 +725,8 @@
DataElement dataElement = dataElementMap.get( trimToNull( dataValue.getDataElement() ) );
Period period = outerPeriod != null ? outerPeriod : PeriodType.getPeriodFromIsoString( trimToNull( dataValue.getPeriod() ) );
- OrganisationUnit orgUnit = outerOrgUnit != null ? outerOrgUnit : orgUnitMap.get( trimToNull( dataValue.getOrgUnit() ) );
+ OrganisationUnit orgUnit = outerOrgUnit != null ? outerOrgUnit : orgUnitMap.get( trimToNull( dataValue.getOrgUnit() ),
+ new IdentifiableObjectCallable<>( identifiableObjectManager, OrganisationUnit.class, trimToNull( dataValue.getOrgUnit() ) ) );
DataElementCategoryOptionCombo categoryOptionCombo = categoryOptionComboMap.get( trimToNull( dataValue.getCategoryOptionCombo() ) );
DataElementCategoryOptionCombo attrOptionCombo = outerAttrOptionCombo != null ? outerAttrOptionCombo :
categoryOptionComboMap.get( trimToNull( dataValue.getAttributeOptionCombo() ) );
=== added file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/CachingMap.java'
--- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/CachingMap.java 1970-01-01 00:00:00 +0000
+++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/CachingMap.java 2015-02-20 11:17:31 +0000
@@ -0,0 +1,60 @@
+package org.hisp.dhis.system.util;
+
+/*
+ * Copyright (c) 2004-2015, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+import java.util.HashMap;
+import java.util.concurrent.Callable;
+
+/**
+ * @author Lars Helge Overland
+ */
+public class CachingMap<K, V>
+ extends HashMap<K, V>
+{
+ public V get( K key, Callable<V> callable )
+ {
+ V value = super.get( key );
+
+ if ( value == null )
+ {
+ try
+ {
+ value = callable.call();
+
+ super.put( key, value );
+ }
+ catch ( Exception ex )
+ {
+ throw new RuntimeException( ex );
+ }
+ }
+
+ return value;
+ }
+}
=== modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/datavalue/ImportDataValueAction.java'
--- dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/datavalue/ImportDataValueAction.java 2015-02-17 06:00:52 +0000
+++ dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/datavalue/ImportDataValueAction.java 2015-02-20 11:17:31 +0000
@@ -128,6 +128,13 @@
this.importFormat = importFormat;
}
+ private boolean preheatCache = true;
+
+ public void setPreheatCache( boolean preheatCache )
+ {
+ this.preheatCache = preheatCache;
+ }
+
// -------------------------------------------------------------------------
// Action implementation
// -------------------------------------------------------------------------
@@ -146,7 +153,7 @@
in = StreamUtils.wrapAndCheckCompressionFormat( in );
- ImportOptions options = new ImportOptions( idScheme, dataElementIdScheme, orgUnitIdScheme, dryRun, true, strategy, skipExistingCheck );
+ ImportOptions options = new ImportOptions( idScheme, dataElementIdScheme, orgUnitIdScheme, dryRun, preheatCache, strategy, skipExistingCheck );
log.info( options );
=== modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/resources/org/hisp/dhis/importexport/i18n_module.properties'
--- dhis-2/dhis-web/dhis-web-importexport/src/main/resources/org/hisp/dhis/importexport/i18n_module.properties 2015-01-03 11:19:11 +0000
+++ dhis-2/dhis-web/dhis-web-importexport/src/main/resources/org/hisp/dhis/importexport/i18n_module.properties 2015-02-20 11:17:31 +0000
@@ -119,6 +119,7 @@
user_name=Username
password=Password
levels=Levels
+preload_cache=Preload cache
import_process_done=Import successful
import_process_failed=Import process failed - See log for details
importing_constants=Importing constants
@@ -346,6 +347,7 @@
metadata_import=Meta-Data Import
metadata_export=Meta-Data Export
gml_import=GML Import
+faster_for_small_imports=Faster for small imports
data_element_or_type_null_or_empty=Data element or type is null or empty
value_length_greater_than_max_length=Value is greater than max length
value_not_numeric=Value is not numeric
=== modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/importDataValue.vm'
--- dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/importDataValue.vm 2014-12-08 21:01:08 +0000
+++ dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/importDataValue.vm 2015-02-20 11:17:31 +0000
@@ -73,6 +73,13 @@
<option value="true">$i18n.getString( "skip_check_fast" )</option>
</select></td>
</tr>
+<tr class="moreOptions" style="display:none">
+ <td>$i18n.getString( "preload_cache" )</td>
+ <td><select id="preheatCache" name="preheatCache" style="width:190px">
+ <option value="true">$i18n.getString( "yes" )</option>
+ <option value="false">$i18n.getString( "no" ) ($i18n.getString( "faster_for_small_imports" ))</option>
+ </select></td>
+</tr>
<tr>
<td></td>
<td><input type="button" value="$i18n.getString( 'import' )" style="width:120px" onclick="importDataValue()"/></td>
