dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #17979
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 7386: DHIS 1.4 import/export enhancements.
------------------------------------------------------------
revno: 7386
committer: Jason P. Pickering <jason.p.pickering@xxxxxxxxx>
branch nick: dhis2
timestamp: Mon 2012-06-25 20:21:40 +0200
message:
DHIS 1.4 import/export enhancements.
modified:
dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/ImportParams.java
dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/converter/DataValueConverter.java
dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/exporter/DefaultDhis14XMLExportService.java
dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/importer/DefaultDhis14XMLImportService.java
dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/imp/ImportAction.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/ImportParams.java'
--- dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/ImportParams.java 2011-12-26 10:07:59 +0000
+++ dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/ImportParams.java 2012-06-25 18:21:40 +0000
@@ -53,6 +53,8 @@
private String minorVersion;
+ private String owner;
+
// -------------------------------------------------------------------------
// Constructors
// -------------------------------------------------------------------------
@@ -61,11 +63,21 @@
{
}
- public ImportParams( ImportType type, ImportStrategy importStrategy, boolean dataValues )
- {
- this.type = type;
- this.importStrategy = importStrategy;
- this.dataValues = dataValues;
+ public ImportParams(ImportType type, ImportStrategy importStrategy, boolean dataValues)
+ {
+ this.type = type;
+ this.importStrategy = importStrategy;
+ this.dataValues = dataValues;
+
+ }
+
+ //Constructor used for DHIS 1.4 imports
+ public ImportParams( ImportType type, ImportStrategy importStrategy, boolean dataValues, String owner )
+ {
+ this.type = type;
+ this.importStrategy = importStrategy;
+ this.dataValues = dataValues;
+ this.owner = owner;
}
// -------------------------------------------------------------------------
@@ -188,4 +200,14 @@
{
this.minorVersion = minorVersion;
}
+
+ public void setOwner(String owner)
+ {
+ this.owner = owner;
+ }
+
+ public String getOwner()
+ {
+ return owner;
+ }
}
=== modified file 'dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/converter/DataValueConverter.java'
--- dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/converter/DataValueConverter.java 2012-06-25 08:34:16 +0000
+++ dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/converter/DataValueConverter.java 2012-06-25 18:21:40 +0000
@@ -33,6 +33,7 @@
import org.hisp.dhis.dataelement.DataElement;
import org.hisp.dhis.dataelement.DataElementCategoryOptionCombo;
import org.hisp.dhis.dataelement.DataElementCategoryService;
+import org.hisp.dhis.dataelement.DataElementService;
import org.hisp.dhis.datavalue.DataValue;
import org.hisp.dhis.datavalue.DeflatedDataValue;
import org.hisp.dhis.importexport.*;
@@ -67,6 +68,8 @@
private DataElementCategoryService categoryService;
private PeriodService periodService;
private StatementManager statementManager;
+ private DataElementService dataElementService;
+
// -------------------------------------------------------------------------
// Properties
@@ -81,11 +84,12 @@
// -------------------------------------------------------------------------
public DataValueConverter( PeriodService periodService, AggregatedDataValueService aggregatedDataValueService,
- StatementManager statementManager )
+ StatementManager statementManager, DataElementService dataElementService )
{
this.periodService = periodService;
this.aggregatedDataValueService = aggregatedDataValueService;
this.statementManager = statementManager;
+ this.dataElementService = dataElementService;
}
/**
@@ -158,12 +162,7 @@
out.write( getCsvValue( value.getSourceId() ) );
out.write( getCsvValue( value.getDataElementId() ) );
out.write( getCsvValue( value.getPeriodId() ) );
- out.write( SEPARATOR_B );
- out.write( SEPARATOR_B );
- out.write( getCsvValue( csvEncode( value.getValue() ) ) );
- out.write( SEPARATOR_B );
- out.write( SEPARATOR_B );
- out.write( SEPARATOR_B );
+ out = getCSVDataExportField(out, value );
out.write( getCsvValue( 0 ) );
out.write( getCsvValue( 0 ) );
out.write( getCsvValue( 0 ) );
@@ -191,7 +190,8 @@
public void read( BufferedReader reader, ImportParams params )
{
String line = "";
-
+ Boolean validValue = true;
+
DataValue value = new DataValue();
DataElement dataElement = new DataElement();
Period period = new Period();
@@ -199,7 +199,8 @@
DataElementCategoryOptionCombo categoryOptionCombo = categoryService.getDefaultDataElementCategoryOptionCombo();
DataElementCategoryOptionCombo proxyCategoryOptionCombo = new DataElementCategoryOptionCombo();
proxyCategoryOptionCombo.setId( categoryOptionCombo.getId() );
-
+ final String owner = params.getOwner();
+
try
{
reader.readLine(); // Skip CSV header
@@ -207,7 +208,7 @@
while( ( line = reader.readLine() ) != null )
{
String[] values = line.split( SEPARATOR );
-
+
dataElement.setId( dataElementMapping.get( Integer.parseInt( values[2] ) ) );
period.setId( periodMapping.get( Integer.parseInt( values[3] ) ) );
organisationUnit.setId( sourceMapping.get( Integer.parseInt( values[1] ) ) );
@@ -215,33 +216,57 @@
value.setDataElement( dataElement );
value.setPeriod( period );
value.setSource( organisationUnit );
- //Text
- if ( values[4] != null || !values[4].isEmpty() )
+
+ if ( values[4] != null || !values[4].isEmpty() ) //Text
{
value.setValue( values[4].trim() );
}
- //Yes=1,No=0
- if ( values[5] != null || !values[5].isEmpty() )
+ else if ( values[5] != null || !values[5].isEmpty() ) //Boolean
{
value.setValue("false");
- if ( values[5].trim() == "1" )
+ if ( values[5].trim().equals("1") )
{
value.setValue("true");
}
}
- //Numbers
- if ( values[6] != null || !values[6].isEmpty() )
- {
- value.setValue( handleNumericValue( values[6] ) );
-
+ else if ( values[6] != null || !values[6].isEmpty() ) //Numeric
+ {
+ value.setValue( handleNumericValue( values[6] ) );
+ validValue = isValidNumeric( value.getValue() );
+ }
+ else if ( values[7] != null || !values[7].isEmpty() ) //Date
+ {
+ try
+ {
+ value.setValue( handleNumericValue( DateUtils.convertDate(values[7]) ) );
+ }
+ catch (Exception e)
+ {
+ validValue = false;
+ }
+
+ }
+ else if ( values[8] != null || !values[8].isEmpty() ) //Memo not supported
+ {
+ validValue = false;
+ }
+
+ else if ( values[9] != null || !values[9].isEmpty() ) //OLE not supported
+ {
+ validValue = false;
}
value.setComment( values[13] );
+ value.setTimestamp( DateUtils.getDefaultDate( values[15] ) );
value.setOptionCombo( proxyCategoryOptionCombo );
+ value.setStoredBy( owner );
- importObject( value, params );
+ if ( validValue )
+ {
+ importObject( value, params );
+ }
}
}
catch ( IOException ex )
@@ -263,50 +288,70 @@
//Remove all quotes
value = value.replaceAll( "\"", "" );
//Strip trailing zeros
- value = value.replaceAll( "\\.0+$", "" );
+ value = value.replaceAll("\\.0+$", "");
}
-
return value;
}
- private static boolean isValidNumeric (String value)
+ private boolean isValidNumeric (String value)
{
- if ( value == null )
- {
- return false;
- }
- else
- {
- return value.matches("-?\\d+(\\.\\d+)?");
- }
+ return value != null && value.matches("-?\\d+(\\.\\d+)?");
}
- private static Integer exportCSVField (DataElement dataElement)
+ private ZipOutputStream getCSVDataExportField (ZipOutputStream out, DeflatedDataValue value)
{
- String dataElementType = dataElement.getType();
- Integer csvField = null;
-
- if ( dataElementType == DataElement.VALUE_TYPE_STRING)
- {
- csvField = 4;
- }
-
- if ( dataElementType == DataElement.VALUE_TYPE_BOOL )
- {
- csvField = 5;
- }
-
- if ( dataElementType == DataElement.VALUE_TYPE_NUMBER )
- {
- csvField = 6;
- }
-
- if ( dataElementType == DataElement.VALUE_TYPE_DATE )
- {
- csvField = 7;
- }
-
- return csvField;
+
+ String dataElementType = dataElementService.getDataElement(value.getDataElementId()).getType();
+
+ try
+ {
+ if ( dataElementType.equals( DataElement.VALUE_TYPE_STRING ) )
+ {
+ out.write( getCsvValue( csvEncode( value.getValue() ) ) );
+ out.write( SEPARATOR_B );
+ out.write( SEPARATOR_B );
+ out.write( SEPARATOR_B );
+ out.write( SEPARATOR_B );
+ out.write( SEPARATOR_B );
+ }
+
+ else if ( dataElementType.equals( DataElement.VALUE_TYPE_BOOL ) )
+ {
+ out.write( SEPARATOR_B );
+ out.write( getCsvValue( csvEncode( value.getValue() ) ) );
+ out.write( SEPARATOR_B );
+ out.write( SEPARATOR_B );
+ out.write( SEPARATOR_B );
+ out.write( SEPARATOR_B );
+ }
+
+ else if ( dataElementType.equals( DataElement.VALUE_TYPE_NUMBER ) )
+ {
+ out.write( SEPARATOR_B );
+ out.write( SEPARATOR_B );
+ out.write( getCsvValue( csvEncode( value.getValue() ) ) );
+ out.write( SEPARATOR_B );
+ out.write( SEPARATOR_B );
+ out.write( SEPARATOR_B );
+ }
+
+ else if ( dataElementType.equals(DataElement.VALUE_TYPE_DATE ) )
+ {
+ out.write( SEPARATOR_B );
+ out.write( SEPARATOR_B );
+ out.write( SEPARATOR_B );
+ out.write( getCsvValue( DateUtils.getMediumDate( value.getValue() ).toString() ) ) ;
+ out.write( SEPARATOR_B );
+ out.write( SEPARATOR_B );
+ }
+ }
+
+ catch ( IOException ex )
+ {
+ throw new RuntimeException( "Failed handle CSV data field export", ex );
+ }
+
+ return out;
}
}
=== modified file 'dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/exporter/DefaultDhis14XMLExportService.java'
--- dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/exporter/DefaultDhis14XMLExportService.java 2012-03-29 21:52:27 +0000
+++ dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/exporter/DefaultDhis14XMLExportService.java 2012-06-25 18:21:40 +0000
@@ -27,14 +27,6 @@
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
-import java.io.BufferedInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PipedInputStream;
-import java.io.PipedOutputStream;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
-
import org.amplecode.quick.StatementManager;
import org.amplecode.staxwax.factory.XMLFactory;
import org.amplecode.staxwax.writer.XMLWriter;
@@ -44,30 +36,16 @@
import org.hisp.dhis.importexport.ExportParams;
import org.hisp.dhis.importexport.ExportPipeThread;
import org.hisp.dhis.importexport.ExportService;
-import org.hisp.dhis.importexport.dhis14.xml.converter.DataElementConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.DataTypeConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.DataValueConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.IndicatorConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.IndicatorTypeConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.OrganisationUnitConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.OrganisationUnitHierarchyConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.PeriodTypeConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.UserConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.UserRoleConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.xsd.DataElementXSDConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.xsd.DataRootXSDConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.xsd.DataTypeXSDConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.xsd.IndicatorTypeXSDConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.xsd.IndicatorXSDConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.xsd.OrganisationUnitHierarchyXSDConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.xsd.OrganisationUnitXSDConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.xsd.PeriodTypeXSDConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.xsd.UserRoleXSDConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.xsd.UserXSDConverter;
+import org.hisp.dhis.importexport.dhis14.xml.converter.*;
+import org.hisp.dhis.importexport.dhis14.xml.converter.xsd.*;
import org.hisp.dhis.indicator.IndicatorService;
import org.hisp.dhis.organisationunit.OrganisationUnitService;
import org.hisp.dhis.period.PeriodService;
+import java.io.*;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
/**
* @author Lars Helge Overland
@@ -195,7 +173,7 @@
thread.registerXMLConverter( new UserConverter() );
thread.registerXMLConverter( new UserRoleConverter() );
- thread.registerCSVConverter( new DataValueConverter( periodService, aggregatedDataValueService, statementManager ) );
+ thread.registerCSVConverter( new DataValueConverter( periodService, aggregatedDataValueService, statementManager, dataElementService ) );
thread.start();
=== modified file 'dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/importer/DefaultDhis14XMLImportService.java'
--- dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/importer/DefaultDhis14XMLImportService.java 2011-12-26 10:07:59 +0000
+++ dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/importer/DefaultDhis14XMLImportService.java 2012-06-25 18:21:40 +0000
@@ -27,12 +27,6 @@
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
-
import org.amplecode.quick.BatchHandler;
import org.amplecode.quick.BatchHandlerFactory;
import org.amplecode.staxwax.factory.XMLFactory;
@@ -45,25 +39,10 @@
import org.hisp.dhis.dataelement.DataElementCategoryService;
import org.hisp.dhis.dataelement.DataElementService;
import org.hisp.dhis.expression.ExpressionService;
-import org.hisp.dhis.importexport.CSVConverter;
-import org.hisp.dhis.importexport.ImportDataValue;
-import org.hisp.dhis.importexport.ImportObjectService;
-import org.hisp.dhis.importexport.ImportParams;
-import org.hisp.dhis.importexport.ImportService;
-import org.hisp.dhis.importexport.XMLConverter;
+import org.hisp.dhis.importexport.*;
import org.hisp.dhis.importexport.analysis.DefaultImportAnalyser;
import org.hisp.dhis.importexport.analysis.ImportAnalyser;
-import org.hisp.dhis.importexport.dhis14.xml.converter.DataElementCategoryComboConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.DataElementCategoryConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.DataElementCategoryOptionComboConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.DataElementCategoryOptionConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.DataElementConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.DataValueConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.IndicatorConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.IndicatorTypeConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.OrganisationUnitConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.OrganisationUnitHierarchyConverter;
-import org.hisp.dhis.importexport.dhis14.xml.converter.PeriodConverter;
+import org.hisp.dhis.importexport.dhis14.xml.converter.*;
import org.hisp.dhis.importexport.invoker.ConverterInvoker;
import org.hisp.dhis.importexport.mapping.NameMappingUtil;
import org.hisp.dhis.importexport.mapping.ObjectMappingGenerator;
@@ -74,6 +53,13 @@
import org.hisp.dhis.system.process.OutputHolderState;
import org.hisp.dhis.system.util.StreamUtils;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
/**
* @author Lars Helge Overland
* @version $Id$
@@ -117,14 +103,14 @@
{
this.periodService = periodService;
}
-
+
private OrganisationUnitService organisationUnitService;
public void setOrganisationUnitService( OrganisationUnitService organisationUnitService )
{
this.organisationUnitService = organisationUnitService;
}
-
+
private ImportObjectService importObjectService;
public void setImportObjectService( ImportObjectService importObjectService )
=== modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/imp/ImportAction.java'
--- dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/imp/ImportAction.java 2011-12-26 10:07:59 +0000
+++ dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/imp/ImportAction.java 2012-06-25 18:21:40 +0000
@@ -235,6 +235,8 @@
// Import parameters
// ---------------------------------------------------------------------
+ String owner = currentUserService.getCurrentUsername();
+
ImportParams params = new ImportParams();
ImportStrategy strategy = ImportStrategy.valueOf( incomingRecords );
@@ -245,6 +247,7 @@
params.setSkipCheckMatching( skipCheckMatching );
params.setLastUpdated( (lastUpdated != null && lastUpdated.trim().length() > 0) ? DateUtils
.getMediumDate( lastUpdated ) : null );
+ params.setOwner(owner);
// ---------------------------------------------------------------------
// Process
@@ -252,8 +255,6 @@
String importType = importFormat + IMPORT_INTERNAL_PROCESS_ID_POSTFIX;
- String owner = currentUserService.getCurrentUsername();
-
ProcessExecutor executor = processCoordinator.newProcess( importType, owner );
ImportInternalProcess importProcess = (ImportInternalProcess) executor.getProcess();
