dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #38467
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 19591: ADX WIP : modified so as not to abort import on individual datavalue conflicts
------------------------------------------------------------
revno: 19591
committer: Bob Jolliffe <bobjolliffe@xxxxxxxxx>
branch nick: dhis2
timestamp: Wed 2015-07-08 17:19:15 +0100
message:
ADX WIP : modified so as not to abort import on individual datavalue conflicts
modified:
dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/ADXException.java
dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/ADXPeriod.java
dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/DefaultADXDataService.java
dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/adx/ADXPeriodTest.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/ADXException.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/ADXException.java 2015-07-03 16:31:04 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/ADXException.java 2015-07-08 16:19:15 +0000
@@ -26,16 +26,38 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
+import org.hisp.dhis.dxf2.importsummary.ImportConflict;
+
/**
- * A simple wrapper for ADX checked exceptions
+ * A simple class for ADX checked exceptions
+ * which can wrap an ImportConflict
*
* @author bobj
*/
public class ADXException
extends Exception
{
+ protected String object;
+
+ public String getObject()
+ {
+ return object;
+ }
+
public ADXException(String msg)
{
super(msg);
}
+
+ // for wrapping ImportConflict
+ public ADXException(String object, String msg)
+ {
+ super(msg);
+ this.object = object;
+ }
+
+ public ImportConflict getImportConflict()
+ {
+ return new ImportConflict(object, this.getMessage());
+ }
}
=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/ADXPeriod.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/ADXPeriod.java 2015-06-16 18:30:37 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/ADXPeriod.java 2015-07-08 16:19:15 +0000
@@ -56,14 +56,6 @@
*/
public class ADXPeriod
{
- public static class ADXPeriodException extends RuntimeException
- {
-
- ADXPeriodException( String message )
- {
- super( message );
- }
- }
public static enum Duration
{
@@ -78,13 +70,13 @@
private static SimpleDateFormat dateFormat = new SimpleDateFormat( "yyyy-MM-dd" );
- public static Period parse( String periodString ) throws ADXPeriodException
+ public static Period parse( String periodString ) throws ADXException
{
String[] tokens = periodString.split( "/" );
if ( tokens.length != 2 )
{
- throw new ADXPeriodException( periodString + " not in valid <date>/<duration> format" );
+ throw new ADXException( periodString + " not in valid <date>/<duration> format" );
}
try
@@ -123,7 +115,7 @@
periodType = new SixMonthlyAprilPeriodType();
break;
default:
- throw new ADXPeriodException( periodString + "is invalid sixmonthly type" );
+ throw new ADXException( periodString + "is invalid sixmonthly type" );
}
case P1Y:
switch ( cal.get( Calendar.MONTH ) )
@@ -141,7 +133,7 @@
periodType = new FinancialOctoberPeriodType();
break;
default:
- throw new ADXPeriodException( periodString + "is invalid yearly type" );
+ throw new ADXException( periodString + "is invalid yearly type" );
}
}
@@ -151,7 +143,7 @@
}
else
{
- throw new ADXPeriodException( "Failed to create period type from " + duration );
+ throw new ADXException( "Failed to create period type from " + duration );
}
return period;
@@ -159,11 +151,11 @@
}
catch ( ParseException ex )
{
- throw new ADXPeriodException( tokens[0] + "is not a valid date in YYYY-MM-dd format" );
+ throw new ADXException( tokens[0] + "is not a valid date in YYYY-MM-dd format" );
}
catch ( IllegalArgumentException ex )
{
- throw new ADXPeriodException( tokens[1] + " is not a supported duration type" );
+ throw new ADXException( tokens[1] + " is not a supported duration type" );
}
}
=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/DefaultADXDataService.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/DefaultADXDataService.java 2015-07-06 08:54:27 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/DefaultADXDataService.java 2015-07-08 16:19:15 +0000
@@ -36,6 +36,7 @@
import java.io.PipedOutputStream;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -68,7 +69,6 @@
import org.hisp.dhis.dataelement.DataElementService;
import org.hisp.dhis.dataset.DataSet;
import org.hisp.dhis.dataset.DataSetService;
-import org.hisp.dhis.dxf2.adx.ADXPeriod.ADXPeriodException;
import org.hisp.dhis.dxf2.common.ImportOptions;
import org.hisp.dhis.dxf2.datavalueset.DataExportParams;
import org.hisp.dhis.dxf2.datavalueset.DataValueSetService;
@@ -79,6 +79,7 @@
import org.hisp.dhis.period.Period;
import org.springframework.beans.factory.annotation.Autowired;
import org.apache.xerces.util.XMLChar;
+import org.hisp.dhis.dxf2.importsummary.ImportConflict;
/**
* @author bobj
@@ -140,6 +141,7 @@
ExecutorService executor = Executors.newSingleThreadExecutor();
+ int count = 0;
// submit each ADX group to DXF importer as a datavalueSet
while ( adxReader.moveToStartElement( ADXConstants.GROUP, ADXConstants.NAMESPACE ) )
{
@@ -149,20 +151,40 @@
futureImportSummary = executor.submit( new PipedImporter( dataValueSetService, importOptions, pipeOut ) );
XMLOutputFactory factory = XMLOutputFactory.newInstance();
XMLStreamWriter dxfWriter = factory.createXMLStreamWriter( pipeOut );
- parseADXGroupToDxf( adxReader, dxfWriter, importOptions );
+
+ // note theis returns conflicts which are detected at adx level
+ List<ImportConflict> adxConflicts = parseADXGroupToDxf( adxReader, dxfWriter, importOptions );
+
pipeOut.flush();
-
- importSummaries.addImportSummary( futureImportSummary.get( TOTAL_MINUTES_TO_WAIT, TimeUnit.MINUTES ) );
- }
- catch ( IOException | XMLStreamException | InterruptedException | ExecutionException | TimeoutException
- | ADXException | ADXPeriodException ex )
- {
- ImportSummary importSummary = new ImportSummary();
- importSummary.setStatus( ImportStatus.ERROR );
- importSummary.setDescription( "Exception: " + ex.getMessage() );
- importSummaries.addImportSummary( importSummary );
- log.warn( "Import failed: " + ex );
- }
+
+ ImportSummary summary = futureImportSummary.get( TOTAL_MINUTES_TO_WAIT, TimeUnit.MINUTES );
+
+ // add adx conflicts to the import summary
+ for ( ImportConflict conflict : adxConflicts)
+ {
+ summary.getConflicts().add( conflict );
+ summary.getImportCount().incrementIgnored();
+ }
+ importSummaries.addImportSummary( summary );
+ }
+ catch ( ADXException ex)
+ {
+ ImportSummary importSummary = new ImportSummary();
+ importSummary.setStatus( ImportStatus.ERROR );
+ importSummary.setDescription( "DataSet import failed for group number " + count );
+ importSummary.getConflicts().add( ex.getImportConflict() );
+ importSummaries.addImportSummary( importSummary );
+ log.warn( "Import failed: " + ex );
+ }
+ catch ( IOException | XMLStreamException | InterruptedException | ExecutionException | TimeoutException ex )
+ {
+ ImportSummary importSummary = new ImportSummary();
+ importSummary.setStatus( ImportStatus.ERROR );
+ importSummary.setDescription( "DataSet import failed for group number " + count );
+ importSummaries.addImportSummary( importSummary );
+ log.warn( "Import failed: " + ex );
+ }
+ count++;
}
executor.shutdown();
@@ -174,9 +196,11 @@
// Utility methods
// -------------------------------------------------------------------------
- private void parseADXGroupToDxf( XMLReader adxReader, XMLStreamWriter dxfWriter, ImportOptions importOptions )
- throws XMLStreamException, ADXException, ADXPeriodException
+ private List<ImportConflict> parseADXGroupToDxf( XMLReader adxReader, XMLStreamWriter dxfWriter, ImportOptions importOptions )
+ throws XMLStreamException, ADXException
{
+ List<ImportConflict> adxConflicts = new LinkedList<>();
+
dxfWriter.writeStartDocument( "1.0" );
dxfWriter.writeStartElement( "dataValueSet" );
dxfWriter.writeDefaultNamespace( "http://dhis2.org/schema/dxf/2.0"; );
@@ -199,7 +223,7 @@
String periodStr = groupAttributes.get( ADXConstants.PERIOD );
groupAttributes.remove( ADXConstants.PERIOD );
Period period = ADXPeriod.parse( periodStr );
- dxfWriter.writeAttribute( "period", period.getIsoDate() );
+ groupAttributes.put( ADXConstants.PERIOD, period.getIsoDate());
// process adx group attributes
if ( !groupAttributes.containsKey( ADXConstants.ATTOPTCOMBO )
@@ -209,6 +233,11 @@
DataSet dataSet = identifiableObjectManager.getObject( DataSet.class, dataElementIdScheme,
groupAttributes.get( ADXConstants.DATASET ) );
+
+ if (dataSet == null)
+ {
+ throw new ADXException("No dataSet matching identifier: " + groupAttributes.get( ADXConstants.DATASET ));
+ }
groupAttributes.put( ADXConstants.DATASET, dataSet.getUid() );
DataElementCategoryCombo attributeCombo = dataSet.getCategoryCombo();
attributesToDXF( ADXConstants.ATTOPTCOMBO, attributeCombo, groupAttributes, dataElementIdScheme );
@@ -223,18 +252,30 @@
// process the dataValues
while ( adxReader.moveToStartElement( ADXConstants.DATAVALUE, ADXConstants.GROUP ) )
{
- parseADXDataValueToDxf( adxReader, dxfWriter, importOptions );
+ try
+ {
+ parseADXDataValueToDxf( adxReader, dxfWriter, importOptions );
+ }
+ catch (ADXException ex)
+ {
+ adxConflicts.add( ex.getImportConflict() );
+ log.info("ADX datavalue conflict: " + ex.getImportConflict());
+ }
}
dxfWriter.writeEndElement();
dxfWriter.writeEndDocument();
+
+ return adxConflicts;
}
private void parseADXDataValueToDxf( XMLReader adxReader, XMLStreamWriter dxfWriter, ImportOptions importOptions )
throws XMLStreamException, ADXException
{
Map<String, String> dvAttributes = readAttributes( adxReader );
-
+
+ log.debug("Processing datavalue: " + dvAttributes );
+
if ( !dvAttributes.containsKey( ADXConstants.DATAELEMENT ) )
{
throw new ADXException( ADXConstants.DATAELEMENT + " attribute is required on 'dataValue'" );
@@ -247,15 +288,12 @@
IdentifiableProperty dataElementIdScheme = importOptions.getDataElementIdScheme();
- dxfWriter.writeStartElement( "dataValue" );
-
DataElement dataElement = identifiableObjectManager.getObject( DataElement.class, dataElementIdScheme,dvAttributes.get( ADXConstants.DATAELEMENT));
- // process adx group attributes
- if ( !dvAttributes.containsKey( ADXConstants.CATOPTCOMBO )
- && dvAttributes.containsKey( ADXConstants.DATASET ) )
+ // process adx datavalue attributes
+ if ( !dvAttributes.containsKey( ADXConstants.CATOPTCOMBO ) )
{
- log.debug( "No attributeOptionCombo present. Check dataSet for attribute categorycombo" );
+ log.debug( "No categoryOptionCombo present." );
DataElementCategoryCombo categoryCombo = dataElement.getCategoryCombo();
attributesToDXF( ADXConstants.CATOPTCOMBO, categoryCombo, dvAttributes, dataElementIdScheme );
@@ -275,12 +313,14 @@
}
}
+ log.debug("Processing datavalue as DXF2: " + dvAttributes );
+
+ dxfWriter.writeStartElement( "dataValue" );
// pass through the remaining attributes to dxf
for ( String attribute : dvAttributes.keySet() )
{
dxfWriter.writeAttribute( attribute, dvAttributes.get( attribute ) );
}
-
dxfWriter.writeEndElement();
}
@@ -321,7 +361,7 @@
}
catch ( CategoryComboMapException ex )
{
- log.warn( "Failed to create catcomboMap from " + catcombo );
+ log.info( "Failed to create catcomboMap from " + catcombo );
throw new ADXException( ex.getMessage() );
}
@@ -333,14 +373,14 @@
if ( catCode == null )
{
- throw new RuntimeException( "No category matching " + catCode );
+ throw new ADXException( "No category matching " + catCode );
}
String catAttribute = attributes.get( catCode );
if ( catAttribute == null )
{
- throw new RuntimeException( "Missing required attribute from catcombo: " + catCode );
+ throw new ADXException( "Missing required attribute from catcombo: " + catCode );
}
compositeIdentifier += "\"" + catAttribute + "\"";
@@ -387,7 +427,7 @@
DataElementCategoryOptionCombo catOptCombo = getCatOptComboFromAttributes( attributeOptions, catCombo, scheme );
attributes.put( optionComboName, catOptCombo.getUid() );
-
+
log.debug( "dxf attributes: " + attributes );
}
=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/adx/ADXPeriodTest.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/adx/ADXPeriodTest.java 2015-06-11 21:37:27 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/adx/ADXPeriodTest.java 2015-07-08 16:19:15 +0000
@@ -49,20 +49,26 @@
@Test
public void testParser()
{
- p = ADXPeriod.parse( "2015-01-01/P1Y" );
- assertEquals( "2015", p.getIsoDate() );
- p = ADXPeriod.parse( "2015-01-01/P1M" );
- assertEquals( "201501", p.getIsoDate() );
- p = ADXPeriod.parse( "2015-01-01/P1D" );
- assertEquals( "20150101", p.getIsoDate() );
- p = ADXPeriod.parse( "2015-01-01/P1Q" );
- assertEquals( "2015Q1", p.getIsoDate() );
- p = ADXPeriod.parse( "2015-04-01/P1Q" );
- assertEquals( "2015Q2", p.getIsoDate() );
- p = ADXPeriod.parse( "2015-01-01/P7D" );
- assertEquals( "2015W1", p.getIsoDate() );
- p = ADXPeriod.parse( "2015-01-05/P7D" );
- assertEquals( "2015W2", p.getIsoDate() );
+ try {
+ p = ADXPeriod.parse( "2015-01-01/P1Y" );
+ assertEquals( "2015", p.getIsoDate() );
+ p = ADXPeriod.parse( "2015-01-01/P1M" );
+ assertEquals( "201501", p.getIsoDate() );
+ p = ADXPeriod.parse( "2015-01-01/P1D" );
+ assertEquals( "20150101", p.getIsoDate() );
+ p = ADXPeriod.parse( "2015-01-01/P1Q" );
+ assertEquals( "2015Q1", p.getIsoDate() );
+ p = ADXPeriod.parse( "2015-04-01/P1Q" );
+ assertEquals( "2015Q2", p.getIsoDate() );
+ p = ADXPeriod.parse( "2015-01-01/P7D" );
+ assertEquals( "2015W1", p.getIsoDate() );
+ p = ADXPeriod.parse( "2015-01-05/P7D" );
+ assertEquals( "2015W2", p.getIsoDate() );
+ }
+ catch (ADXException ex)
+ {
+ fail(ex.getMessage());
+ }
}
@Test
@@ -75,21 +81,28 @@
}
catch ( Exception ex )
{
- assertEquals( ADXPeriod.ADXPeriodException.class, ex.getClass() );
+ assertEquals( ADXException.class, ex.getClass() );
}
}
@Test
public void testFinancialTypes()
{
- p = ADXPeriod.parse( "2015-01-01/P1Y" );
- assertEquals( "2015", p.getIsoDate() );
- p = ADXPeriod.parse( "2015-04-01/P1Y" );
- assertEquals( "2015April", p.getIsoDate() );
- p = ADXPeriod.parse( "2015-07-01/P1Y" );
- assertEquals( "2015July", p.getIsoDate() );
- p = ADXPeriod.parse( "2015-10-01/P1Y" );
- assertEquals( "2015Oct", p.getIsoDate() );
+ try
+ {
+ p = ADXPeriod.parse( "2015-01-01/P1Y" );
+ assertEquals( "2015", p.getIsoDate() );
+ p = ADXPeriod.parse( "2015-04-01/P1Y" );
+ assertEquals( "2015April", p.getIsoDate() );
+ p = ADXPeriod.parse( "2015-07-01/P1Y" );
+ assertEquals( "2015July", p.getIsoDate() );
+ p = ADXPeriod.parse( "2015-10-01/P1Y" );
+ assertEquals( "2015Oct", p.getIsoDate() );
+ }
+ catch (ADXException ex)
+ {
+ fail(ex.getMessage());
+ }
}
@Test
@@ -102,7 +115,7 @@
}
catch ( Exception ex )
{
- assertEquals( ADXPeriod.ADXPeriodException.class, ex.getClass() );
+ assertEquals( ADXException.class, ex.getClass() );
}
}
