dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #28154
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 14041: First cut of data element CSV import
------------------------------------------------------------
revno: 14041
committer: Lars Helge Øverland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Sat 2014-02-15 19:09:14 +0100
message:
First cut of data element CSV import
added:
dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/utils/CsvObjectUtils.java
dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/util/ImportMetaDataCsvTask.java
modified:
dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/dxf2/MetaDataImportAction.java
dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/util/ImportMetaDataTask.java
dhis-2/dhis-web/dhis-web-importexport/src/main/resources/META-INF/dhis/beans.xml
dhis-2/dhis-web/dhis-web-importexport/src/main/resources/org/hisp/dhis/importexport/i18n_module.properties
dhis-2/dhis-web/dhis-web-importexport/src/main/resources/struts.xml
dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/dxf2MetaDataImport.vm
dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/mainMenu.vm
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== added file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/utils/CsvObjectUtils.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/utils/CsvObjectUtils.java 1970-01-01 00:00:00 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/utils/CsvObjectUtils.java 2014-02-15 18:09:14 +0000
@@ -0,0 +1,102 @@
+package org.hisp.dhis.dxf2.utils;
+
+/*
+ * Copyright (c) 2004-2013, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+import org.hisp.dhis.common.CodeGenerator;
+import org.hisp.dhis.dataelement.DataElement;
+import org.hisp.dhis.dxf2.metadata.MetaData;
+
+import com.csvreader.CsvReader;
+
+/**
+ * @author Lars Helge Overland
+ */
+public class CsvObjectUtils
+{
+ public static MetaData fromCsv( InputStream input, Class<?> clazz )
+ throws IOException
+ {
+ CsvReader reader = new CsvReader( input, Charset.forName( "UTF-8" ) );
+
+ reader.readRecord(); // Ignore first row
+
+ List<DataElement> dataElements = new ArrayList<DataElement>();
+
+ while ( reader.readRecord() )
+ {
+ String[] values = reader.getValues();
+
+ if ( values == null || values.length == 0 )
+ {
+ continue;
+ }
+
+ DataElement element = new DataElement();
+ element.setName( getSafe( values, 0, null ) );
+ element.setUid( getSafe( values, 1, CodeGenerator.generateCode() ) );
+ element.setCode( getSafe( values, 2, null ) );
+ element.setShortName( getSafe( values, 3, StringUtils.substring( element.getName(), 0, 50 ) ) );
+ element.setDescription( getSafe( values, 4, null ) );
+ element.setFormName( getSafe( values, 5, null ) );
+ element.setActive( Boolean.valueOf( getSafe( values, 6, "false" ) ) );
+ element.setDomainType( getSafe( values, 7, DataElement.DOMAIN_TYPE_AGGREGATE ) );
+ element.setType( getSafe( values, 8, DataElement.VALUE_TYPE_INT ) );
+ element.setNumberType( getSafe( values, 9, DataElement.VALUE_TYPE_NUMBER ) );
+ element.setTextType( getSafe( values, 10, null ) );
+ element.setAggregationOperator( getSafe( values, 11, DataElement.AGGREGATION_OPERATOR_SUM ) );
+ element.setUrl( getSafe( values, 12, null ) );
+ element.setZeroIsSignificant( Boolean.valueOf( getSafe( values, 13, "false" ) ) );
+
+ dataElements.add( element );
+ }
+
+ MetaData metaData = new MetaData();
+ metaData.setDataElements( dataElements );
+ return metaData;
+ }
+
+ private static final String getSafe( String[] values, int index, String defaultValue )
+ {
+ if ( values == null || index < 0 || index >= values.length )
+ {
+ return null;
+ }
+
+ String value = values[index];
+
+ return value != null ? value : defaultValue;
+ }
+}
=== modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/dxf2/MetaDataImportAction.java'
--- dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/dxf2/MetaDataImportAction.java 2013-08-23 16:05:01 +0000
+++ dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/dxf2/MetaDataImportAction.java 2014-02-15 18:09:14 +0000
@@ -32,6 +32,7 @@
import org.hisp.dhis.dxf2.metadata.ImportOptions;
import org.hisp.dhis.dxf2.metadata.ImportService;
import org.hisp.dhis.importexport.ImportStrategy;
+import org.hisp.dhis.importexport.action.util.ImportMetaDataCsvTask;
import org.hisp.dhis.importexport.action.util.ImportMetaDataTask;
import org.hisp.dhis.scheduling.TaskCategory;
import org.hisp.dhis.scheduling.TaskId;
@@ -93,6 +94,18 @@
this.strategy = ImportStrategy.valueOf( strategy );
}
+ private String importFormat;
+
+ public String getImportFormat()
+ {
+ return importFormat;
+ }
+
+ public void setImportFormat( String importFormat )
+ {
+ this.importFormat = importFormat;
+ }
+
// -------------------------------------------------------------------------
// Action Implementation
// -------------------------------------------------------------------------
@@ -115,8 +128,17 @@
importOptions.setStrategy( strategy.toString() );
importOptions.setDryRun( dryRun );
- scheduler.executeTask( new ImportMetaDataTask( ( user != null ? user.getUid() : null ), importService, importOptions, in, taskId ) );
-
+ String userId = user != null ? user.getUid() : null;
+
+ if ( "csv".equals( importFormat ) )
+ {
+ scheduler.executeTask( new ImportMetaDataCsvTask( userId, importService, importOptions, in, taskId ) );
+ }
+ else
+ {
+ scheduler.executeTask( new ImportMetaDataTask( userId, importService, importOptions, in, taskId ) );
+ }
+
return SUCCESS;
}
}
=== added file 'dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/util/ImportMetaDataCsvTask.java'
--- dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/util/ImportMetaDataCsvTask.java 1970-01-01 00:00:00 +0000
+++ dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/util/ImportMetaDataCsvTask.java 2014-02-15 18:09:14 +0000
@@ -0,0 +1,88 @@
+package org.hisp.dhis.importexport.action.util;
+
+/*
+ * Copyright (c) 2004-2013, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.hisp.dhis.dataelement.DataElement;
+import org.hisp.dhis.dxf2.metadata.ImportOptions;
+import org.hisp.dhis.dxf2.metadata.ImportService;
+import org.hisp.dhis.dxf2.metadata.MetaData;
+import org.hisp.dhis.dxf2.utils.CsvObjectUtils;
+import org.hisp.dhis.scheduling.TaskId;
+
+/**
+ * @author Morten Olav Hansen <mortenoh@xxxxxxxxx>
+ */
+public class ImportMetaDataCsvTask
+ implements Runnable
+{
+ private static final Log log = LogFactory.getLog( ImportMetaDataTask.class );
+
+ private ImportService importService;
+
+ private ImportOptions importOptions;
+
+ private InputStream inputStream;
+
+ private TaskId taskId;
+
+ private String userUid;
+
+ public ImportMetaDataCsvTask( String userUid, ImportService importService, ImportOptions importOptions, InputStream inputStream,
+ TaskId taskId )
+ {
+ this.importService = importService;
+ this.importOptions = importOptions;
+ this.inputStream = inputStream;
+ this.taskId = taskId;
+ this.userUid = userUid;
+ }
+
+ @Override
+ public void run()
+ {
+ MetaData metaData = null;
+
+ try
+ {
+ metaData = CsvObjectUtils.fromCsv( inputStream, DataElement.class );
+ }
+ catch ( IOException ex )
+ {
+ log.error( "Unable to read meta-data while reading input stream", ex );
+ return;
+ }
+
+ importService.importMetaData( userUid, metaData, importOptions, taskId );
+ }
+}
=== modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/util/ImportMetaDataTask.java'
--- dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/util/ImportMetaDataTask.java 2013-08-23 16:05:01 +0000
+++ dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/util/ImportMetaDataTask.java 2014-02-15 18:09:14 +0000
@@ -70,11 +70,11 @@
@Override
public void run()
{
- MetaData metaData;
+ MetaData metaData = null;
try
{
- // TODO should probably sniff if its xml or json, but this works for now
+ // TODO sniff if its xml or json, but this works for now
metaData = JacksonUtils.fromXml( inputStream, MetaData.class );
}
catch ( IOException ignored )
=== modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/resources/META-INF/dhis/beans.xml'
--- dhis-2/dhis-web/dhis-web-importexport/src/main/resources/META-INF/dhis/beans.xml 2014-02-04 09:38:57 +0000
+++ dhis-2/dhis-web/dhis-web-importexport/src/main/resources/META-INF/dhis/beans.xml 2014-02-15 18:09:14 +0000
@@ -222,7 +222,6 @@
class="org.hisp.dhis.importexport.action.event.GetImportEventSummariesAction" scope="prototype">
</bean>
- <!--@author Ovidiu Rosu <rosu.ovi@xxxxxxxxx>-->
<bean id="org.hisp.dhis.importexport.action.dxf2.FilterListAction"
class="org.hisp.dhis.importexport.action.dxf2.FilterListAction" scope="prototype">
<property name="metaDataFilterService" ref="org.hisp.dhis.filter.MetaDataFilterService" />
=== modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/resources/org/hisp/dhis/importexport/i18n_module.properties'
--- dhis-2/dhis-web/dhis-web-importexport/src/main/resources/org/hisp/dhis/importexport/i18n_module.properties 2014-02-05 12:00:46 +0000
+++ dhis-2/dhis-web/dhis-web-importexport/src/main/resources/org/hisp/dhis/importexport/i18n_module.properties 2014-02-15 18:09:14 +0000
@@ -330,6 +330,8 @@
check_safe_recommended=Check (safe, recommended)
skip_check_fast=Skip check (fast)
dxf2=DXF2
+xml_metadata_import=XML Meta-Data Import
+csv_metadata_import=CSV Meta-Data Import
metadata_import=Meta-Data Import
metadata_export=Meta-Data Export
gml_import=GML Import
=== modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/resources/struts.xml'
--- dhis-2/dhis-web/dhis-web-importexport/src/main/resources/struts.xml 2014-02-04 09:38:57 +0000
+++ dhis-2/dhis-web/dhis-web-importexport/src/main/resources/struts.xml 2014-02-15 18:09:14 +0000
@@ -215,7 +215,6 @@
<param name="requiredAuthorities">F_EXPORT_DATA</param>
</action>
- <!-- New -->
<action name="displayDataExportForm" class="org.hisp.dhis.importexport.action.NoAction">
<result name="success" type="velocity">/main.vm</result>
<param name="page">/dhis-web-importexport/exportDataForm.vm</param>
@@ -224,7 +223,6 @@
<param name="requiredAuthorities">F_EXPORT_DATA</param>
</action>
-
<action name="displayDetailedMetaDataExportForm"
class="org.hisp.dhis.importexport.action.exp.GetDetailedExportOptionsAction">
<result name="success" type="velocity">/main.vm</result>
@@ -244,7 +242,6 @@
</result>
</action>
- <!-- New -->
<action name="exportData" class="org.hisp.dhis.importexport.action.exp.DataExportAction">
<result name="success" type="stream">
<param name="contentType">application/zip</param>
@@ -322,7 +319,7 @@
</action>
<action name="importMetaData" class="org.hisp.dhis.importexport.action.dxf2.MetaDataImportAction">
- <result name="success" type="redirect">dxf2MetaDataImport.action</result>
+ <result name="success" type="redirect">dxf2MetaDataImport.action?importFormat=${importFormat}</result>
<interceptor-ref name="fileUploadStack" />
</action>
=== modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/dxf2MetaDataImport.vm'
--- dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/dxf2MetaDataImport.vm 2013-05-23 09:39:06 +0000
+++ dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/dxf2MetaDataImport.vm 2014-02-15 18:09:14 +0000
@@ -1,8 +1,8 @@
-<h3>$i18n.getString( "metadata_import" )</h3>
+<h3>$!i18n.getString( $importFormat ) $i18n.getString( "metadata_import" )</h3>
<div id="inputCriteria" class="inputCriteria" style="width: 380px">
-<form id="importForm" name="importForm" method="post" enctype="multipart/form-data" action="importMetaData.action">
+<form id="importForm" name="importForm" method="post" enctype="multipart/form-data" action="importMetaData.action?importFormat=$!{importFormat}">
<table>
<col width="140">
<col>
=== modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/mainMenu.vm'
--- dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/mainMenu.vm 2013-10-30 12:51:03 +0000
+++ dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/mainMenu.vm 2014-02-15 18:09:14 +0000
@@ -2,7 +2,8 @@
<h2>$i18n.getString( "import" ) </h2>
<ul>
- <li><a href="dxf2MetaDataImport.action">$i18n.getString( "metadata_import" ) </a></li>
+ <li><a href="dxf2MetaDataImport.action?importFormat=xml">$i18n.getString( "xml_metadata_import" ) </a></li>
+ <li><a href="dxf2MetaDataImport.action?importFormat=csv">$i18n.getString( "csv_metadata_import" ) </a></li>
<li><a href="displayImportDataValueForm.action?importFormat=xml">$i18n.getString( "xml_data_import" ) </a></li>
<li><a href="displayImportDataValueForm.action?importFormat=csv">$i18n.getString( "csv_data_import" ) </a></li>
<li><a href="displayImportDataValueForm.action?importFormat=pdf">$i18n.getString( "pdf_data_import" ) </a></li>
