dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #36889
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 18868: Implemented support for matching on uid and code (as well as name) in GML import. Matching is pri...
Merge authors:
Halvdan Hoem Grelland (halvdanhg)
------------------------------------------------------------
revno: 18868 [merge]
committer: Halvdan Hoem Grelland <halvdanhg@xxxxxxxxx>
branch nick: dhis2
timestamp: Fri 2015-04-10 01:44:18 +0200
message:
Implemented support for matching on uid and code (as well as name) in GML import. Matching is prioritized in order [uid, code, name]. Basic implementation only, still needs more rigorous testing (and possibly integration test cases).
modified:
dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitService.java
dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitStore.java
dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitService.java
dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/hibernate/HibernateOrganisationUnitStore.java
dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/gml/DefaultGmlImportService.java
dhis-2/dhis-services/dhis-service-dxf2/src/main/resources/gml/gml2dxf2.xsl
dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/util/ImportMetaDataGmlTask.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitService.java'
--- dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitService.java 2015-03-15 20:49:24 +0000
+++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitService.java 2015-03-31 17:17:03 +0000
@@ -208,6 +208,14 @@
Collection<OrganisationUnit> getOrganisationUnitsByNames( Collection<String> names );
/**
+ * Returns all OrganisationUnits matching the given codes.
+ *
+ * @param codes codes of OrganisationUnits to return.
+ * @return the OrganisationUnits matching the given codes.
+ */
+ Collection<OrganisationUnit> getOrganisationUnitsByCodes( Collection<String> codes );
+
+ /**
* Returns all root OrganisationUnits. A root OrganisationUnit is an
* OrganisationUnit with no parent/the parent set to null.
*
=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitStore.java'
--- dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitStore.java 2015-01-17 07:41:26 +0000
+++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitStore.java 2015-03-31 17:17:03 +0000
@@ -67,6 +67,14 @@
Collection<OrganisationUnit> getByNames( Collection<String> names );
/**
+ * Retrieves all OrganisationUnits matching the given codes.
+ *
+ * @param codes codes of the OrganisationUnits to return.
+ * @return all OrganisationUnits matching the given codes.
+ */
+ Collection<OrganisationUnit> getByCodes( Collection<String> codes );
+
+ /**
* Returns all OrganisationUnits by status.
*
* @param active Get active or inactive
=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitService.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitService.java 2015-03-15 20:49:24 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitService.java 2015-03-31 17:17:03 +0000
@@ -271,12 +271,19 @@
return organisationUnitStore.getAllEqNameIgnoreCase( name );
}
- @Override public Collection<OrganisationUnit> getOrganisationUnitsByNames( Collection<String> names )
+ @Override
+ public Collection<OrganisationUnit> getOrganisationUnitsByNames( Collection<String> names )
{
return i18n( i18nService, organisationUnitStore.getByNames( names ) );
}
@Override
+ public Collection<OrganisationUnit> getOrganisationUnitsByCodes( Collection<String> codes )
+ {
+ return i18n( i18nService, organisationUnitStore.getByCodes( codes ) );
+ }
+
+ @Override
public Collection<OrganisationUnit> getRootOrganisationUnits()
{
return i18n( i18nService, organisationUnitStore.getRootOrganisationUnits());
=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/hibernate/HibernateOrganisationUnitStore.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/hibernate/HibernateOrganisationUnitStore.java 2015-02-19 09:18:17 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/hibernate/HibernateOrganisationUnitStore.java 2015-04-08 15:29:00 +0000
@@ -31,6 +31,7 @@
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashSet;
@@ -92,6 +93,11 @@
@SuppressWarnings( "unchecked" )
public Collection<OrganisationUnit> getByNames( Collection<String> names )
{
+ if ( names == null || names.isEmpty() )
+ {
+ return new ArrayList<>();
+ }
+
Query query = getQuery( "from OrganisationUnit where name in :names" );
query.setParameterList( "names", names );
@@ -99,6 +105,22 @@
}
@Override
+ @SuppressWarnings( "unchecked" )
+ public Collection<OrganisationUnit> getByCodes( Collection<String> codes )
+ {
+
+ if ( codes == null || codes.isEmpty() )
+ {
+ return new ArrayList<>();
+ }
+
+ Query query = getQuery( "from OrganisationUnit where code in :codes" );
+ query.setParameterList( "codes", codes );
+
+ return query.list();
+ }
+
+ @Override
@SuppressWarnings("unchecked")
public Collection<OrganisationUnit> getAllOrganisationUnitsByStatus( boolean active )
{
=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/gml/DefaultGmlImportService.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/gml/DefaultGmlImportService.java 2015-03-05 16:05:19 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/gml/DefaultGmlImportService.java 2015-04-09 23:35:37 +0000
@@ -29,7 +29,11 @@
*/
import com.google.common.base.Function;
+import com.google.common.base.Strings;
+import com.google.common.collect.Iterators;
import com.google.common.collect.Maps;
+import org.hisp.dhis.common.IdentifiableObjectManager;
+import org.hisp.dhis.common.IdentifiableProperty;
import org.hisp.dhis.common.MergeStrategy;
import org.hisp.dhis.dxf2.common.ImportOptions;
import org.hisp.dhis.dxf2.metadata.ImportService;
@@ -51,6 +55,9 @@
import java.io.IOException;
import java.io.InputStream;
import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
import java.util.Map;
/**
@@ -74,6 +81,9 @@
@Autowired
private OrganisationUnitService organisationUnitService;
+ @Autowired
+ private IdentifiableObjectManager idObjectManager;
+
// -------------------------------------------------------------------------
// GmlImportService implementation
// -------------------------------------------------------------------------
@@ -83,48 +93,43 @@
throws IOException, TransformerException
{
InputStream dxfStream = transformGml( inputStream );
-
MetaData metaData = renderService.fromXml( dxfStream, MetaData.class );
-
dxfStream.close();
- Map<String, OrganisationUnit> namedMap = Maps.uniqueIndex( metaData.getOrganisationUnits(),
- new Function<OrganisationUnit, String>()
- {
- @Override
- public String apply( OrganisationUnit organisationUnit )
- {
- return organisationUnit.getName();
- }
- }
- );
-
- // Fetch persisted OrganisationUnits and merge imported GML properties
- Collection<OrganisationUnit> persistedOrgUnits = organisationUnitService.getOrganisationUnitsByNames( namedMap.keySet() );
-
- for( OrganisationUnit persisted : persistedOrgUnits )
+ Map<String, OrganisationUnit> uidMap = Maps.newHashMap(), codeMap = Maps.newHashMap(), nameMap = Maps.newHashMap();
+
+ matchAndFilterOnIdentifiers( metaData.getOrganisationUnits(), uidMap, codeMap, nameMap );
+
+ Map<String, OrganisationUnit> persistedUidMap = getMatchingPersistedOrgUnits( uidMap.keySet(), IdentifiableProperty.UID );
+ Map<String, OrganisationUnit> persistedCodeMap = getMatchingPersistedOrgUnits( codeMap.keySet(), IdentifiableProperty.CODE );
+ Map<String, OrganisationUnit> persistedNameMap = getMatchingPersistedOrgUnits( nameMap.keySet(), IdentifiableProperty.NAME );
+
+ Iterator<OrganisationUnit> persistedIterator = Iterators.concat( persistedUidMap.values().iterator(),
+ persistedCodeMap.values().iterator(), persistedNameMap.values().iterator() );
+
+ while ( persistedIterator.hasNext() )
{
- OrganisationUnit unit = namedMap.get( persisted.getName() );
-
- if( unit == null || unit.getCoordinates() == null || unit.getFeatureType() == null )
- {
- continue;
- }
-
- String coordinates = unit.getCoordinates(),
- featureType = unit.getFeatureType();
-
- unit.mergeWith( persisted, MergeStrategy.MERGE_IF_NOT_NULL );
-
- unit.setCoordinates( coordinates );
- unit.setFeatureType( featureType );
-
- if( persisted.getParent() != null )
- {
- OrganisationUnit parent = new OrganisationUnit();
- parent.setUid( persisted.getParent().getUid() );
- unit.setParent( parent );
- }
+ OrganisationUnit persisted = persistedIterator.next(), imported = null;
+
+ if ( !Strings.isNullOrEmpty( persisted.getUid() ) && uidMap.containsKey( persisted.getUid() ) )
+ {
+ imported = uidMap.get( persisted.getUid() );
+ }
+ else if ( !Strings.isNullOrEmpty( persisted.getCode() ) && codeMap.containsKey( persisted.getCode() ) )
+ {
+ imported = codeMap.get( persisted.getCode() );
+ }
+ else if ( !Strings.isNullOrEmpty( persisted.getName() ) && nameMap.containsKey( persisted.getName() ) )
+ {
+ imported = nameMap.get( persisted.getName() );
+ }
+
+ if ( imported == null || imported.getCoordinates() == null || imported.getFeatureType() == null )
+ {
+ continue; // Failed to dereference a persisted entity for this org unit or geo data incomplete/missing, therefore ignore
+ }
+
+ mergeNonGeoData( persisted, imported );
}
return metaData;
@@ -157,4 +162,65 @@
return new ByteArrayInputStream( output.toByteArray() );
}
+
+ private void matchAndFilterOnIdentifiers( List<OrganisationUnit> sourceList, Map<String, OrganisationUnit> uidMap, Map<String,
+ OrganisationUnit> codeMap, Map<String, OrganisationUnit> nameMap )
+ {
+ for ( OrganisationUnit orgUnit : sourceList ) // Identifier Matching priority: uid, code, name
+ {
+ // Only matches if UID is actually in DB as an empty UID on input will be replaced by auto-generated value
+ if ( !Strings.isNullOrEmpty( orgUnit.getUid() ) && idObjectManager.exists( OrganisationUnit.class, orgUnit.getUid() ) )
+ {
+ uidMap.put( orgUnit.getUid(), orgUnit );
+ }
+ else if ( !Strings.isNullOrEmpty( orgUnit.getCode() ) )
+ {
+ codeMap.put( orgUnit.getCode(), orgUnit );
+ }
+ else if ( !Strings.isNullOrEmpty( orgUnit.getName() ) )
+ {
+ nameMap.put( orgUnit.getName(), orgUnit );
+ }
+ }
+ }
+
+ private Map<String, OrganisationUnit> getMatchingPersistedOrgUnits( Collection<String> identifiers, final IdentifiableProperty idProperty )
+ {
+ Collection<OrganisationUnit> orgUnits =
+ idProperty == IdentifiableProperty.UID ? organisationUnitService.getOrganisationUnitsByUid( identifiers ) :
+ idProperty == IdentifiableProperty.CODE ? organisationUnitService.getOrganisationUnitsByCodes( identifiers ) :
+ idProperty == IdentifiableProperty.NAME ? organisationUnitService.getOrganisationUnitsByNames( identifiers ) :
+ new HashSet<OrganisationUnit>();
+
+ return Maps.uniqueIndex( orgUnits,
+ new Function<OrganisationUnit, String>()
+ {
+ @Override
+ public String apply( OrganisationUnit organisationUnit )
+ {
+ return idProperty == IdentifiableProperty.UID ? organisationUnit.getUid() :
+ idProperty == IdentifiableProperty.CODE ? organisationUnit.getCode() :
+ idProperty == IdentifiableProperty.NAME ? organisationUnit.getName() : null;
+ }
+ }
+ );
+ }
+
+ private void mergeNonGeoData( OrganisationUnit source, OrganisationUnit target )
+ {
+ String coordinates = target.getCoordinates(),
+ featureType = target.getFeatureType();
+
+ target.mergeWith( source, MergeStrategy.MERGE );
+
+ target.setCoordinates( coordinates );
+ target.setFeatureType( featureType );
+
+ if ( source.getParent() != null )
+ {
+ OrganisationUnit parent = new OrganisationUnit();
+ parent.setUid( source.getParent().getUid() );
+ target.setParent( parent );
+ }
+ }
}
=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/resources/gml/gml2dxf2.xsl'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/resources/gml/gml2dxf2.xsl 2015-03-24 16:52:51 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/resources/gml/gml2dxf2.xsl 2015-04-09 23:35:37 +0000
@@ -78,14 +78,27 @@
</xsl:template>
<xsl:template match="gml:featureMember">
- <xsl:variable name="name" select=".//*[local-name()='Name' or local-name()='NAME' or local-name()='name']"/>
+ <xsl:variable name="uid" select=".//*[local-name()='uid' or local-name()='UID' or local-name()='Uid']" />
+ <xsl:variable name="code" select=".//*[local-name()='code' or local-name()='CODE' or local-name()='Code']" />
+ <xsl:variable name="name" select=".//*[local-name()='name' or local-name()='NAME' or local-name()='Name']" />
<organisationUnit>
- <xsl:attribute name="name">
- <xsl:value-of select="$name"/>
- </xsl:attribute>
- <xsl:attribute name="shortName">
- <xsl:value-of select="substring($name,1,50)"/>
- </xsl:attribute>
+ <xsl:choose> <!-- Priority is uid, code, name. First match in order excludes the others -->
+ <xsl:when test="$uid != ''">
+ <xsl:attribute name="id"> <!-- 'uid' is mapped to 'id' in dxf2 -->
+ <xsl:value-of select="$uid" />
+ </xsl:attribute>
+ </xsl:when>
+ <xsl:when test="$code != ''">
+ <xsl:attribute name="code">
+ <xsl:value-of select="$code" />
+ </xsl:attribute>
+ </xsl:when>
+ <xsl:when test="$name != ''">
+ <xsl:attribute name="name">
+ <xsl:value-of select="$name" />
+ </xsl:attribute>
+ </xsl:when>
+ </xsl:choose>
<xsl:apply-templates select="./child::node()/child::node()/gml:Polygon|./child::node()/child::node()/gml:MultiPolygon|./child::node()/child::node()/gml:Point"/>
<active>true</active>
</organisationUnit>
=== modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/util/ImportMetaDataGmlTask.java'
--- dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/util/ImportMetaDataGmlTask.java 2015-02-17 06:00:52 +0000
+++ dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/util/ImportMetaDataGmlTask.java 2015-04-09 14:51:07 +0000
@@ -88,7 +88,7 @@
{
gmlImportService.importGml( inputStream, userUid, importOptions, taskId );
}
- catch ( IOException | TransformerException e)
+ catch ( IOException | TransformerException e )
{
log.error( "Unable to read GML data from input stream", e );
}
