dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #37781
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 19296: WIP proper error reporting on failed GML parsing during import
------------------------------------------------------------
revno: 19296
committer: Halvdan Hoem Grelland <halvdanhg@xxxxxxxxx>
branch nick: dhis2
timestamp: Fri 2015-06-05 14:56:34 +0200
message:
WIP proper error reporting on failed GML parsing during import
added:
dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/gml/GmlPreProcessingResult.java
modified:
dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/gml/DefaultGmlImportService.java
dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/gml/GmlImportService.java
dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/gml/GmlImportServiceTest.java
dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/SpringScheduler.java
dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/dxf2/MetaDataImportAction.java
dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/util/ImportMetaDataGmlTask.java
dhis-2/dhis-web/dhis-web-importexport/src/main/resources/org/hisp/dhis/importexport/i18n_module.properties
dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/importMetaDataSummary.vm
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/gml/DefaultGmlImportService.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/gml/DefaultGmlImportService.java 2015-06-03 15:00:19 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/gml/DefaultGmlImportService.java 2015-06-05 12:56:34 +0000
@@ -32,6 +32,8 @@
import com.google.common.base.Strings;
import com.google.common.collect.Iterators;
import com.google.common.collect.Maps;
+import org.apache.commons.io.IOUtils;
+import org.hibernate.Hibernate;
import org.hisp.dhis.common.IdentifiableObjectManager;
import org.hisp.dhis.common.IdentifiableProperty;
import org.hisp.dhis.common.MergeStrategy;
@@ -88,13 +90,91 @@
// GmlImportService implementation
// -------------------------------------------------------------------------
+ @Transactional
+ @Override
+ public GmlPreProcessingResult preProcessGml( InputStream inputStream )
+ {
+ InputStream dxfStream = null;
+ MetaData metaData = null;
+
+ try
+ {
+ dxfStream = transformGml( inputStream );
+ metaData = renderService.fromXml( dxfStream, MetaData.class );
+ }
+ catch ( IOException | TransformerException e )
+ {
+ return GmlPreProcessingResult.failure( e );
+ }
+ finally
+ {
+ IOUtils.closeQuietly( dxfStream );
+ }
+
+ Map<String, OrganisationUnit> uidMap = Maps.newHashMap(), codeMap = Maps.newHashMap(), nameMap = Maps.newHashMap();
+
+ matchAndFilterOnIdentifiers( metaData.getOrganisationUnits(), uidMap, codeMap, nameMap );
+
+ Map<String, OrganisationUnit> persistedUidMap = getMatchingPersistedOrgUnits( uidMap.keySet(), IdentifiableProperty.UID );
+ Map<String, OrganisationUnit> persistedCodeMap = getMatchingPersistedOrgUnits( codeMap.keySet(), IdentifiableProperty.CODE );
+ Map<String, OrganisationUnit> persistedNameMap = getMatchingPersistedOrgUnits( nameMap.keySet(), IdentifiableProperty.NAME );
+
+ Iterator<OrganisationUnit> persistedIterator = Iterators.concat( persistedUidMap.values().iterator(),
+ persistedCodeMap.values().iterator(), persistedNameMap.values().iterator() );
+
+ while ( persistedIterator.hasNext() )
+ {
+ OrganisationUnit persisted = persistedIterator.next(), imported = null;
+
+ if ( !Strings.isNullOrEmpty( persisted.getUid() ) && uidMap.containsKey( persisted.getUid() ) )
+ {
+ imported = uidMap.get( persisted.getUid() );
+ }
+ else if ( !Strings.isNullOrEmpty( persisted.getCode() ) && codeMap.containsKey( persisted.getCode() ) )
+ {
+ imported = codeMap.get( persisted.getCode() );
+ }
+ else if ( !Strings.isNullOrEmpty( persisted.getName() ) && nameMap.containsKey( persisted.getName() ) )
+ {
+ imported = nameMap.get( persisted.getName() );
+ }
+
+ if ( imported == null || imported.getCoordinates() == null || imported.getFeatureType() == null )
+ {
+ continue; // Failed to dereference a persisted entity for this org unit or geo data incomplete/missing, therefore ignore
+ }
+
+ mergeNonGeoData( persisted, imported );
+ }
+
+ return GmlPreProcessingResult.success( metaData );
+ }
+
@Override
public MetaData fromGml( InputStream inputStream )
throws IOException, TransformerException
{
- InputStream dxfStream = transformGml( inputStream );
- MetaData metaData = renderService.fromXml( dxfStream, MetaData.class );
- dxfStream.close();
+ InputStream dxfStream;
+ MetaData metaData;
+
+ try
+ {
+ dxfStream = transformGml( inputStream );
+ }
+ catch (Exception e)
+ {
+ dxfStream = null;
+ }
+
+ if(dxfStream != null)
+ {
+ metaData = renderService.fromXml( dxfStream, MetaData.class );
+ dxfStream.close();
+ }
+ else
+ {
+ return null;
+ }
Map<String, OrganisationUnit> uidMap = Maps.newHashMap(), codeMap = Maps.newHashMap(), nameMap = Maps.newHashMap();
@@ -143,6 +223,7 @@
importService.importMetaData( userUid, fromGml( inputStream ), importOptions, taskId );
}
+ @Transactional
@Override
public void importGml( MetaData metaData, String userUid, ImportOptions importOptions, TaskId taskId )
{
=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/gml/GmlImportService.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/gml/GmlImportService.java 2015-06-03 15:00:19 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/gml/GmlImportService.java 2015-06-05 12:56:34 +0000
@@ -62,6 +62,24 @@
throws IOException, TransformerException;
/**
+ * Pre-process a GML document. The process, in short, entails the following:
+ * <ol>
+ * <li>Parse the GML payload and transform it into DXF2 format</li>
+ * <li>Get the given identifiers (uid, code or name) from the parsed payload and fetch
+ * the corresponding entities from the DB</li>
+ * <li>Merge the geospatial data given in the input GML into DB entities and return</li>
+ * </ol>
+ *
+ * The result of this process in returned in a {@link GmlPreProcessingResult} which
+ * encapsulates the returned {@link MetaData} object or the exception in cause of parse
+ * failure due to IO errors or malformed input.
+ *
+ * @param gmlInputStream the InputStream providing the GML input.
+ * @return a GmlPreProcessingResult representing the end result of the process.
+ */
+ GmlPreProcessingResult preProcessGml( InputStream gmlInputStream );
+
+ /**
* Imports GML data and merges the geospatial data updates into the database.
* See {@link #fromGml(InputStream)} for details on the underlying process.
*
=== added file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/gml/GmlPreProcessingResult.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/gml/GmlPreProcessingResult.java 1970-01-01 00:00:00 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/gml/GmlPreProcessingResult.java 2015-06-05 12:56:34 +0000
@@ -0,0 +1,70 @@
+package org.hisp.dhis.dxf2.gml;
+
+import org.hisp.dhis.dxf2.metadata.MetaData;
+
+import java.io.InputStream;
+
+/**
+ * Wraps the result of {@link GmlImportService#preProcessGml(InputStream)}.
+ * This is necessary when performing GML import on a context where exceptions
+ * due to malformed input cannot be caught, thus leaving the user uninformed of
+ * the error. This class will wrap the failure and provide the Throwable to the
+ * consuming class on error or the resulting MetaData object on success.
+ *
+ * @author Halvdan Hoem grelland <halvdanhg@xxxxxxxxx>
+ */
+public final class GmlPreProcessingResult
+{
+ private boolean isSuccess;
+ private MetaData resultMetaData;
+ private Throwable throwable;
+
+ private GmlPreProcessingResult(){}
+
+ public static GmlPreProcessingResult success( MetaData resultMetaData ) {
+ GmlPreProcessingResult result = new GmlPreProcessingResult();
+ result.setResultMetaData( resultMetaData );
+ result.setSuccess( true );
+
+ return result;
+ }
+
+ public static GmlPreProcessingResult failure( Throwable throwable )
+ {
+ GmlPreProcessingResult result = new GmlPreProcessingResult();
+ result.setThrowable( throwable );
+ result.setSuccess( false );
+
+ return result;
+ }
+
+ private void setSuccess( boolean isSuccess )
+ {
+ this.isSuccess = isSuccess;
+ }
+
+ public boolean isSuccess()
+ {
+ return isSuccess;
+ }
+
+ private void setResultMetaData( MetaData resultMetaData )
+ {
+ this.resultMetaData = resultMetaData;
+ }
+
+ public MetaData getResultMetaData()
+ {
+ return resultMetaData;
+ }
+
+ private void setThrowable( Throwable throwable )
+ {
+ this.throwable = throwable;
+ }
+
+ public Throwable getThrowable()
+ {
+ return throwable;
+ }
+}
=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/gml/GmlImportServiceTest.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/gml/GmlImportServiceTest.java 2015-01-17 07:41:26 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/gml/GmlImportServiceTest.java 2015-06-05 12:56:34 +0000
@@ -117,4 +117,6 @@
assertEquals( 1, units.get( "Blindern").getCoordinatesAsList().get( 0 ).getNumberOfCoordinates() );
assertEquals( 76, units.get( "Forskningsparken" ).getCoordinatesAsList().get(0).getNumberOfCoordinates() );
}
+
+ // TODO Add test for GmlImportService#preProcessGml(InputStream)
}
=== modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/SpringScheduler.java'
--- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/SpringScheduler.java 2015-05-30 13:36:07 +0000
+++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/SpringScheduler.java 2015-06-05 12:56:34 +0000
@@ -77,7 +77,7 @@
{
taskExecutor.execute( task );
}
-
+
@Override
public boolean scheduleTask( String key, Runnable task, String cronExpr )
{
=== modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/dxf2/MetaDataImportAction.java'
--- dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/dxf2/MetaDataImportAction.java 2015-05-28 16:10:07 +0000
+++ dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/dxf2/MetaDataImportAction.java 2015-06-05 12:56:34 +0000
@@ -187,7 +187,7 @@
}
else if ( "gml".equals( importFormat ) )
{
- scheduler.executeTask( new ImportMetaDataGmlTask( userId, gmlImportService, importOptions, in, taskId ) );
+ scheduler.executeTask( new ImportMetaDataGmlTask( userId, gmlImportService, notifier, importOptions, in, taskId ) );
}
else if ( "json".equals( importFormat ) || "xml".equals( importFormat ) )
{
=== modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/util/ImportMetaDataGmlTask.java'
--- dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/util/ImportMetaDataGmlTask.java 2015-04-11 14:06:51 +0000
+++ dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/util/ImportMetaDataGmlTask.java 2015-06-05 12:56:34 +0000
@@ -28,14 +28,18 @@
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
+import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.hisp.dhis.dxf2.common.ImportOptions;
import org.hisp.dhis.dxf2.gml.GmlImportService;
+import org.hisp.dhis.dxf2.gml.GmlPreProcessingResult;
import org.hisp.dhis.scheduling.TaskId;
+import org.hisp.dhis.system.notification.NotificationLevel;
+import org.hisp.dhis.system.notification.Notifier;
+import org.springframework.web.util.HtmlUtils;
+import org.xml.sax.SAXParseException;
-import javax.xml.transform.TransformerException;
-import java.io.IOException;
import java.io.InputStream;
/**
@@ -50,12 +54,15 @@
private String userUid;
+
// -------------------------------------------------------------------------
// Dependencies
// -------------------------------------------------------------------------
private GmlImportService gmlImportService;
+ private Notifier notifier;
+
private ImportOptions importOptions;
private InputStream inputStream;
@@ -64,11 +71,12 @@
// Constructors
// -------------------------------------------------------------------------
- public ImportMetaDataGmlTask( String userUid, GmlImportService gmlImportService,
+ public ImportMetaDataGmlTask( String userUid, GmlImportService gmlImportService, Notifier notifier,
ImportOptions importOptions, InputStream inputStream, TaskId taskId )
{
this.userUid = userUid;
this.gmlImportService = gmlImportService;
+ this.notifier = notifier;
this.importOptions = importOptions;
this.inputStream = inputStream;
this.taskId = taskId;
@@ -83,15 +91,37 @@
{
importOptions.setImportStrategy( "update" ); // Force update only for GML import
- try
- {
- gmlImportService.importGml( inputStream, userUid, importOptions, taskId );
- }
- catch ( IOException | TransformerException e )
- {
- log.error( "Unable to read GML data from input stream", e );
-
- throw new RuntimeException( "Failed to parse GML input stream", e );
- }
+ GmlPreProcessingResult gmlPreProcessingResult = gmlImportService.preProcessGml( inputStream );
+
+ if ( !gmlPreProcessingResult.isSuccess() )
+ {
+ Throwable throwable = gmlPreProcessingResult.getThrowable();
+ String message = createErrorMessage( throwable );
+
+ notifier.notify( taskId, NotificationLevel.ERROR, message, false );
+ log.error( "GML import failed: " + message, throwable );
+
+ return;
+ }
+
+ gmlImportService.importGml( gmlPreProcessingResult.getResultMetaData(), userUid, importOptions, taskId );
+ }
+
+ private String createErrorMessage( Throwable throwable )
+ {
+ String message = "";
+ Throwable rootThrowable = ExceptionUtils.getRootCause( throwable );
+
+ if ( rootThrowable instanceof SAXParseException )
+ {
+ SAXParseException e = (SAXParseException) rootThrowable;
+ message += "Syntax error on line " + e.getLineNumber() + ". " + e.getMessage();
+ }
+ else
+ {
+ message += rootThrowable.getMessage();
+ }
+
+ return HtmlUtils.htmlEscape( message );
}
}
=== modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/resources/org/hisp/dhis/importexport/i18n_module.properties'
--- dhis-2/dhis-web/dhis-web-importexport/src/main/resources/org/hisp/dhis/importexport/i18n_module.properties 2015-02-20 11:17:31 +0000
+++ dhis-2/dhis-web/dhis-web-importexport/src/main/resources/org/hisp/dhis/importexport/i18n_module.properties 2015-06-05 12:56:34 +0000
@@ -300,6 +300,7 @@
ignored=Ignored
conflicts=Conflicts
no_conflicts_found=No conflicts found
+no_import_summary_available=No summary available
type=Type
count=Count
export_as_xml=Export as XML
=== modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/importMetaDataSummary.vm'
--- dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/importMetaDataSummary.vm 2013-03-21 09:15:24 +0000
+++ dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/importMetaDataSummary.vm 2015-06-05 12:56:34 +0000
@@ -1,4 +1,6 @@
<h3>$i18n.getString( "import_summary" )</h3>
+
+#if( $summary ) ## ImportSummary can be null on failed GML import (pre-processing fails)
<h4>$i18n.getString( "import_count" )</h4>
$summary.importCount.imported Imported<br/>
@@ -54,3 +56,6 @@
#else
<p>$i18n.getString( "no_conflicts_found" )</p>
#end
+#else
+ $i18n.getString( "no_summary_available" )
+#end
\ No newline at end of file
Follow ups
