dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #41111
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 20929: ADX. Added ADX import resource at /api/dataValueSets with content type application/xml+adx. Added...
------------------------------------------------------------
revno: 20929
committer: Lars Helge Overland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Mon 2015-11-02 13:56:53 -0500
message:
ADX. Added ADX import resource at /api/dataValueSets with content type application/xml+adx. Added support for ADX import through UI.
modified:
dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/AdxDataService.java
dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/DefaultAdxDataService.java
dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/PipedImporter.java
dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/tasks/ImportDataValueTask.java
dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataValueSetController.java
dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/datavalue/ImportDataValueAction.java
dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/importDataValue.vm
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/AdxDataService.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/AdxDataService.java 2015-11-02 17:18:46 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/AdxDataService.java 2015-11-02 18:56:53 +0000
@@ -71,7 +71,7 @@
*
* @param in the InputStream.
* @param importOptions the importOptions.
- * @param id the task id.
+ * @param id the task id, can be null.
*
* @return an ImportSummaries collection of ImportSummary for each DataValueSet.
*/
=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/DefaultAdxDataService.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/DefaultAdxDataService.java 2015-11-02 17:18:46 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/DefaultAdxDataService.java 2015-11-02 18:56:53 +0000
@@ -80,6 +80,7 @@
import org.hisp.dhis.scheduling.TaskId;
import org.hisp.dhis.system.notification.Notifier;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.transaction.annotation.Transactional;
/**
* @author bobj
@@ -128,6 +129,7 @@
}
@Override
+ @Transactional
public ImportSummaries saveDataValueSet( InputStream in, ImportOptions importOptions, TaskId id )
{
notifier.clear( id ).notify( id, "ADX parsing process started" );
@@ -148,7 +150,7 @@
try ( PipedOutputStream pipeOut = new PipedOutputStream() )
{
Future<ImportSummary> futureImportSummary;
- futureImportSummary = executor.submit( new PipedImporter( dataValueSetService, importOptions, pipeOut ) );
+ futureImportSummary = executor.submit( new PipedImporter( dataValueSetService, importOptions, id, pipeOut ) );
XMLOutputFactory factory = XMLOutputFactory.newInstance();
XMLStreamWriter dxfWriter = factory.createXMLStreamWriter( pipeOut );
@@ -165,13 +167,14 @@
summary.getConflicts().add( conflict );
summary.getImportCount().incrementIgnored();
}
+
importSummaries.addImportSummary( summary );
}
catch ( AdxException ex)
{
ImportSummary importSummary = new ImportSummary();
importSummary.setStatus( ImportStatus.ERROR );
- importSummary.setDescription( "DataSet import failed for group number " + count );
+ importSummary.setDescription( "Data set import failed for group number: " + count );
importSummary.getConflicts().add( ex.getImportConflict() );
importSummaries.addImportSummary( importSummary );
log.warn( "Import failed: " + ex );
@@ -180,7 +183,7 @@
{
ImportSummary importSummary = new ImportSummary();
importSummary.setStatus( ImportStatus.ERROR );
- importSummary.setDescription( "DataSet import failed for group number " + count );
+ importSummary.setDescription( "Data set import failed for group number: " + count );
importSummaries.addImportSummary( importSummary );
log.warn( "Import failed: " + ex );
}
@@ -190,8 +193,6 @@
executor.shutdown();
- notifier.notify( id, "ADX parsing done" );
-
return importSummaries;
}
@@ -222,7 +223,7 @@
throw new AdxException( AdxDataService.ORGUNIT + " attribute is required on 'group'" );
}
- // translate ADX period to dxf2
+ // translate ADX period to DXF
String periodStr = groupAttributes.get( AdxDataService.PERIOD );
groupAttributes.remove( AdxDataService.PERIOD );
Period period = AdxPeriod.parse( periodStr );
@@ -232,14 +233,14 @@
if ( !groupAttributes.containsKey( AdxDataService.ATTOPTCOMBO )
&& groupAttributes.containsKey( AdxDataService.DATASET ) )
{
- log.debug( "No attributeOptionCombo present. Check dataSet for attribute categorycombo" );
+ log.debug( "No attribute option combo present, check data set for attribute category combo" );
DataSet dataSet = identifiableObjectManager.getObject( DataSet.class, dataElementIdScheme,
groupAttributes.get( AdxDataService.DATASET ) );
- if (dataSet == null)
+ if ( dataSet == null )
{
- throw new AdxException("No dataSet matching identifier: " + groupAttributes.get( AdxDataService.DATASET ));
+ throw new AdxException("No data set matching identifier: " + groupAttributes.get( AdxDataService.DATASET ) );
}
groupAttributes.put( AdxDataService.DATASET, dataSet.getUid() );
@@ -263,7 +264,8 @@
catch (AdxException ex)
{
adxConflicts.add( ex.getImportConflict() );
- log.info("ADX datavalue conflict: " + ex.getImportConflict());
+
+ log.info("ADX data value conflict: " + ex.getImportConflict() );
}
}
@@ -278,7 +280,7 @@
{
Map<String, String> dvAttributes = readAttributes( adxReader );
- log.debug("Processing datavalue: " + dvAttributes );
+ log.debug( "Processing data value: " + dvAttributes );
if ( !dvAttributes.containsKey( AdxDataService.DATAELEMENT ) )
{
@@ -292,17 +294,18 @@
IdentifiableProperty dataElementIdScheme = importOptions.getDataElementIdScheme();
- DataElement dataElement = identifiableObjectManager.getObject( DataElement.class, dataElementIdScheme,dvAttributes.get( AdxDataService.DATAELEMENT));
+ DataElement dataElement = identifiableObjectManager.getObject( DataElement.class, dataElementIdScheme, dvAttributes.get( AdxDataService.DATAELEMENT ) );
if ( dataElement == null )
{
- throw new AdxException(dvAttributes.get( AdxDataService.DATAELEMENT), "No matching dataelement");
+ throw new AdxException(dvAttributes.get( AdxDataService.DATAELEMENT), "No matching dataelement" );
}
// process ADX data value attributes
if ( !dvAttributes.containsKey( AdxDataService.CATOPTCOMBO ) )
{
- log.debug( "No categoryOptionCombo present." );
+ log.debug( "No category option combo present" );
+
DataElementCategoryCombo categoryCombo = dataElement.getCategoryCombo();
attributesToDxf( AdxDataService.CATOPTCOMBO, categoryCombo, dvAttributes, dataElementIdScheme );
@@ -312,6 +315,7 @@
if ( dataElement.getValueType().isText() )
{
adxReader.moveToStartElement( AdxDataService.ANNOTATION, AdxDataService.DATAVALUE );
+
if ( adxReader.isStartElement( AdxDataService.ANNOTATION ) )
{
String textValue = adxReader.getElementValue();
@@ -323,7 +327,7 @@
}
}
- log.debug("Processing datavalue as DXF2: " + dvAttributes );
+ log.debug( "Processing data value as DXF: " + dvAttributes );
dxfWriter.writeStartElement( "dataValue" );
@@ -332,15 +336,16 @@
{
dxfWriter.writeAttribute( attribute, dvAttributes.get( attribute ) );
}
+
dxfWriter.writeEndElement();
}
- private Map<String, DataElementCategory> createCategoryMap( DataElementCategoryCombo catcombo )
+ private Map<String, DataElementCategory> createCategoryMap( DataElementCategoryCombo categoryCombo )
throws AdxException
{
Map<String, DataElementCategory> categoryMap = new HashMap<>();
- List<DataElementCategory> categories = catcombo.getCategories();
+ List<DataElementCategory> categories = categoryCombo.getCategories();
for ( DataElementCategory category : categories )
{
@@ -380,38 +385,38 @@
for ( DataElementCategory category : catcomboMap.getCategories() )
{
- String catCode = category.getCode();
+ String categoryCode = category.getCode();
- if ( catCode == null )
+ if ( categoryCode == null )
{
- throw new AdxException( "No category matching " + catCode );
+ throw new AdxException( "No category matching: " + categoryCode );
}
- String catAttribute = attributes.get( catCode );
+ String catAttribute = attributes.get( categoryCode );
if ( catAttribute == null )
{
- throw new AdxException( "Missing required attribute from catcombo: " + catCode );
+ throw new AdxException( "Missing required attribute from catcombo: " + categoryCode );
}
compositeIdentifier += "\"" + catAttribute + "\"";
}
- DataElementCategoryOptionCombo catoptcombo = catcomboMap.getCategoryOptionCombo( compositeIdentifier );
+ DataElementCategoryOptionCombo catOptionCombo = catcomboMap.getCategoryOptionCombo( compositeIdentifier );
- if ( catoptcombo == null )
+ if ( catOptionCombo == null )
{
throw new AdxException( "Invalid attributes:" + attributes );
}
- return catoptcombo;
+ return catOptionCombo;
}
private void attributesToDxf( String optionComboName, DataElementCategoryCombo catCombo,
Map<String, String> attributes, IdentifiableProperty scheme )
throws AdxException
{
- log.debug( "adx attributes: " + attributes );
+ log.debug( "ADX attributes: " + attributes );
if ( catCombo.isDefault() )
{
@@ -431,14 +436,15 @@
}
else
{
- throw new AdxException( "catcombo " + catCombo.getName() + " must have " + categoryMap.get( category ).getName() );
+ throw new AdxException( "Category combo " + catCombo.getName() + " must have " + categoryMap.get( category ).getName() );
}
}
DataElementCategoryOptionCombo catOptCombo = getCatOptComboFromAttributes( attributeOptions, catCombo, scheme );
+
attributes.put( optionComboName, catOptCombo.getUid() );
- log.debug( "dxf attributes: " + attributes );
+ log.debug( "DXF attributes: " + attributes );
}
// TODO this should be part of staxwax library
=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/PipedImporter.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/PipedImporter.java 2015-10-05 17:45:17 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/PipedImporter.java 2015-11-02 18:56:53 +0000
@@ -37,6 +37,7 @@
import org.hisp.dhis.dxf2.common.ImportOptions;
import org.hisp.dhis.dxf2.importsummary.ImportStatus;
import org.hisp.dhis.dxf2.importsummary.ImportSummary;
+import org.hisp.dhis.scheduling.TaskId;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
@@ -56,14 +57,17 @@
private final ImportOptions importOptions;
+ private final TaskId id;
+
private final Authentication authentication;
public PipedImporter( DataValueSetService dataValueSetService, ImportOptions importOptions,
- PipedOutputStream pipeOut ) throws IOException
+ TaskId id, PipedOutputStream pipeOut ) throws IOException
{
this.dataValueSetService = dataValueSetService;
this.pipeIn = new PipedInputStream( pipeOut, PIPE_BUFFER_SIZE );
this.importOptions = importOptions;
+ this.id = id;
this.authentication = SecurityContextHolder.getContext().getAuthentication();
}
@@ -75,7 +79,7 @@
try
{
- result = dataValueSetService.saveDataValueSet( pipeIn, importOptions );
+ result = dataValueSetService.saveDataValueSet( pipeIn, importOptions, id );
}
catch ( Exception ex )
{
=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/tasks/ImportDataValueTask.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/tasks/ImportDataValueTask.java 2015-10-31 03:58:41 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/tasks/ImportDataValueTask.java 2015-11-02 18:56:53 +0000
@@ -29,6 +29,7 @@
*/
import org.hisp.dhis.security.SecurityContextRunnable;
+import org.hisp.dhis.dxf2.adx.AdxDataService;
import org.hisp.dhis.dxf2.common.ImportOptions;
import org.hisp.dhis.dxf2.datavalueset.DataValueSetService;
import org.hisp.dhis.scheduling.TaskId;
@@ -45,22 +46,27 @@
public static final String FORMAT_JSON = "json";
public static final String FORMAT_CSV = "csv";
public static final String FORMAT_PDF = "pdf";
+ public static final String FORMAT_ADX = "adx";
private DataValueSetService dataValueSetService;
+ private AdxDataService adxDataService;
+
private InputStream inputStream;
- private final ImportOptions options;
+ private final ImportOptions importOptions;
private final TaskId taskId;
private final String format;
- public ImportDataValueTask( DataValueSetService dataValueSetService, InputStream inputStream, ImportOptions options, TaskId taskId, String format )
+ public ImportDataValueTask( DataValueSetService dataValueSetService, AdxDataService adxDataService,
+ InputStream inputStream, ImportOptions importOptions, TaskId taskId, String format )
{
this.dataValueSetService = dataValueSetService;
+ this.adxDataService = adxDataService;
this.inputStream = inputStream;
- this.options = options;
+ this.importOptions = importOptions;
this.taskId = taskId;
this.format = format;
}
@@ -70,19 +76,23 @@
{
if ( FORMAT_JSON.equals( format ) )
{
- dataValueSetService.saveDataValueSetJson( inputStream, options, taskId );
+ dataValueSetService.saveDataValueSetJson( inputStream, importOptions, taskId );
}
else if ( FORMAT_CSV.equals( format ) )
{
- dataValueSetService.saveDataValueSetCsv( inputStream, options, taskId );
+ dataValueSetService.saveDataValueSetCsv( inputStream, importOptions, taskId );
}
else if ( FORMAT_PDF.equals( format ) )
{
- dataValueSetService.saveDataValueSetPdf( inputStream, options, taskId );
+ dataValueSetService.saveDataValueSetPdf( inputStream, importOptions, taskId );
+ }
+ else if ( FORMAT_ADX.equals( format ) )
+ {
+ adxDataService.saveDataValueSet( inputStream, importOptions, taskId );
}
else // FORMAT_XML
{
- dataValueSetService.saveDataValueSet( inputStream, options, taskId );
+ dataValueSetService.saveDataValueSet( inputStream, importOptions, taskId );
}
}
}
=== modified file 'dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataValueSetController.java'
--- dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataValueSetController.java 2015-11-02 05:14:38 +0000
+++ dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataValueSetController.java 2015-11-02 18:56:53 +0000
@@ -1,5 +1,7 @@
package org.hisp.dhis.webapi.controller;
+import org.hisp.dhis.dxf2.adx.AdxDataService;
+
/*
* Copyright (c) 2004-2015, University of Oslo
* All rights reserved.
@@ -33,6 +35,7 @@
import org.hisp.dhis.dxf2.datavalueset.DataExportParams;
import org.hisp.dhis.dxf2.datavalueset.DataValueSetService;
import org.hisp.dhis.dxf2.datavalueset.tasks.ImportDataValueTask;
+import org.hisp.dhis.dxf2.importsummary.ImportSummaries;
import org.hisp.dhis.dxf2.importsummary.ImportSummary;
import org.hisp.dhis.dxf2.render.RenderService;
import org.hisp.dhis.scheduling.TaskCategory;
@@ -76,6 +79,9 @@
private DataValueSetService dataValueSetService;
@Autowired
+ private AdxDataService adxDataService;
+
+ @Autowired
private RenderService renderService;
@Autowired
@@ -171,6 +177,24 @@
}
}
+ @RequestMapping( method = RequestMethod.POST, consumes = "application/xml+adx" )
+ @PreAuthorize( "hasRole('ALL') or hasRole('F_DATAVALUE_ADD')" )
+ public void postAdxDataValueSet( ImportOptions importOptions,
+ HttpServletRequest request, HttpServletResponse response ) throws IOException
+ {
+ if ( importOptions.isAsync() )
+ {
+ startAsyncImport( importOptions, ImportDataValueTask.FORMAT_ADX, request, response );
+ }
+ else
+ {
+ ImportSummaries summaries = adxDataService.saveDataValueSet( request.getInputStream(), importOptions, null );
+
+ response.setContentType( CONTENT_TYPE_XML );
+ renderService.toXml( response.getOutputStream(), summaries );
+ }
+ }
+
@RequestMapping( method = RequestMethod.POST, consumes = "application/json" )
@PreAuthorize( "hasRole('ALL') or hasRole('F_DATAVALUE_ADD')" )
public void postJsonDataValueSet( ImportOptions importOptions,
@@ -226,7 +250,7 @@
InputStream inputStream = saveTmp( request.getInputStream() );
TaskId taskId = new TaskId( TaskCategory.DATAVALUE_IMPORT, currentUserService.getCurrentUser() );
- scheduler.executeTask( new ImportDataValueTask( dataValueSetService, inputStream, importOptions, taskId, format ) );
+ scheduler.executeTask( new ImportDataValueTask( dataValueSetService, adxDataService, inputStream, importOptions, taskId, format ) );
response.setHeader( "Location", ContextUtils.getRootPath( request ) + "/system/tasks/" + TaskCategory.DATAVALUE_IMPORT );
response.setStatus( HttpServletResponse.SC_ACCEPTED );
=== modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/datavalue/ImportDataValueAction.java'
--- dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/datavalue/ImportDataValueAction.java 2015-10-31 03:58:41 +0000
+++ dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/datavalue/ImportDataValueAction.java 2015-11-02 18:56:53 +0000
@@ -36,6 +36,7 @@
import org.apache.commons.logging.LogFactory;
import org.hisp.dhis.common.IdentifiableProperty;
import org.hisp.dhis.commons.util.StreamUtils;
+import org.hisp.dhis.dxf2.adx.AdxDataService;
import org.hisp.dhis.dxf2.common.ImportOptions;
import org.hisp.dhis.dxf2.datavalueset.DataValueSetService;
import org.hisp.dhis.dxf2.datavalueset.tasks.ImportDataValueTask;
@@ -59,6 +60,9 @@
@Autowired
private DataValueSetService dataValueSetService;
+
+ @Autowired
+ private AdxDataService adxDataService;
@Autowired
private CurrentUserService currentUserService;
@@ -160,7 +164,7 @@
log.info( options );
- scheduler.executeTask( new ImportDataValueTask( dataValueSetService, in, options, taskId, importFormat ) );
+ scheduler.executeTask( new ImportDataValueTask( dataValueSetService, adxDataService, in, options, taskId, importFormat ) );
return SUCCESS;
}
=== modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/importDataValue.vm'
--- dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/importDataValue.vm 2015-08-30 19:03:15 +0000
+++ dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/importDataValue.vm 2015-11-02 18:56:53 +0000
@@ -18,6 +18,7 @@
<option value="json">JSON</option>
<option value="csv">CSV</option>
<option value="pdf">PDF</option>
+ <option value="adx">ADX</option>
</select>
</td>
</tr>
