dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #13866
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 4561: Some minor bug fixes in StaxDataValueImportService. Plus:
------------------------------------------------------------
revno: 4561
committer: Bob Jolliffe bobjolliffe@xxxxxxxxx
branch nick: dhis2
timestamp: Sun 2011-09-11 17:32:01 +0100
message:
Some minor bug fixes in StaxDataValueImportService. Plus:
Changed from direct use of batchhandler to use DataValueImporter
Wired up xmlPreConverter so that it can output dxf2 as well as dxf1
modified:
dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/DefaultImportService.java
dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dxf2/service/StaXDataValueImportService.java
dhis-2/dhis-services/dhis-service-importexport/src/main/resources/META-INF/dhis/beans.xml
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/DefaultImportService.java'
--- dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/DefaultImportService.java 2011-09-09 16:30:37 +0000
+++ dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/DefaultImportService.java 2011-09-11 16:32:01 +0000
@@ -158,7 +158,7 @@
XMLReader dxfReader = null;
File transformOutput = null;
- FileInputStream dxfInStream = null;
+ BufferedInputStream dxfInStream = null;
try
{
@@ -186,8 +186,11 @@
transformOutStream.close();
log.info( "Transform successful" );
- dxfInStream = new FileInputStream( transformOutput );
+ dxfInStream =
+ new BufferedInputStream(new FileInputStream( transformOutput ));
+ rootNameSpace = preConverter.getDocumentRoot( dxfInStream ).getNamespaceURI();
dxfReader = XMLFactory.getXMLReader( dxfInStream );
+
}
if (rootNameSpace.equals( DXF2URI )) {
=== modified file 'dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dxf2/service/StaXDataValueImportService.java'
--- dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dxf2/service/StaXDataValueImportService.java 2011-09-09 11:20:37 +0000
+++ dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dxf2/service/StaXDataValueImportService.java 2011-09-11 16:32:01 +0000
@@ -38,12 +38,13 @@
import static org.apache.commons.lang.StringUtils.defaultIfEmpty;
import javax.xml.namespace.QName;
-
import org.amplecode.quick.BatchHandler;
+
import org.amplecode.quick.BatchHandlerFactory;
import org.amplecode.staxwax.reader.XMLReader;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.hisp.dhis.aggregation.AggregatedDataValueService;
import org.hisp.dhis.common.ProcessState;
import org.hisp.dhis.dataelement.DataElement;
import org.hisp.dhis.dataelement.DataElementCategoryOptionCombo;
@@ -52,9 +53,11 @@
import org.hisp.dhis.dataset.DataSet;
import org.hisp.dhis.dataset.DataSetService;
import org.hisp.dhis.datavalue.DataValue;
+import org.hisp.dhis.datavalue.DataValueService;
import org.hisp.dhis.importexport.ImportException;
import org.hisp.dhis.importexport.ImportParams;
import org.hisp.dhis.importexport.dxf2.model.*;
+import org.hisp.dhis.importexport.importer.DataValueImporter;
import org.hisp.dhis.jdbc.batchhandler.DataValueBatchHandler;
import org.hisp.dhis.organisationunit.OrganisationUnit;
import org.hisp.dhis.organisationunit.OrganisationUnitService;
@@ -94,10 +97,14 @@
public static final String SUCCESS = "DataValue import complete";
+ public static final String COUNTER = "%s DataValues imported";
+
+ // update display frequency
+ public static final int DISPLAYCOUNT = 1000;
+
// -------------------------------------------------------------------------
// Dependencies
// -------------------------------------------------------------------------
-
private CurrentUserService currentUserService;
public void setCurrentUserService( CurrentUserService currentUserService )
@@ -105,6 +112,13 @@
this.currentUserService = currentUserService;
}
+ private AggregatedDataValueService aggregatedDataValueService;
+
+ public void setAggregatedDataValueService( AggregatedDataValueService aggregatedDataValueService )
+ {
+ this.aggregatedDataValueService = aggregatedDataValueService;
+ }
+
private DataValueSetService dataValueSetService;
public void setDataValueSetService( DataValueSetService dataValueSetService )
@@ -119,6 +133,13 @@
this.dataSetService = dataSetService;
}
+ private DataValueService dataValueService;
+
+ public void setDataValueService( DataValueService dataValueService )
+ {
+ this.dataValueService = dataValueService;
+ }
+
private DataElementService dataElementService;
public void setDataElementService( DataElementService dataElementService )
@@ -158,9 +179,12 @@
{
String user = currentUserService.getCurrentUsername();
- BatchHandler<DataValue> batchHandler = batchHandlerFactory.createBatchHandler(
+ BatchHandler batchHandler = batchHandlerFactory.createBatchHandler(
DataValueBatchHandler.class ).init();
+ DataValueImporter importer =
+ new DataValueImporter(batchHandler, aggregatedDataValueService, params);
+
int cumulativeDataValueCounter = 0;
try
@@ -192,16 +216,25 @@
// process datavaluesets until no more datavaluesets
int countDataValueSets = 0;
+ int displayCount = DISPLAYCOUNT;
do
{
// look for a DataValue set
- reader.moveToStartElement( Dxf.DATAVALUESET );
+ try
+ {
+ reader.moveToStartElement( Dxf.DATAVALUESET );
+ } catch ( java.util.NoSuchElementException ex )
+ {
+ // we have to reach here eventuallyperiodId
+ break;
+ }
if ( !reader.isStartElement( Dxf.DATAVALUESET ) )
{
// we have to reach here eventually
break;
}
+
// Pick off the attributes
String idSchemeStr = reader.getAttributeValue( DataValueSet.ATTR_IDSCHEME );
String dataSet = reader.getAttributeValue( DataValueSet.ATTR_DATASET );
@@ -229,8 +262,7 @@
}
}
- int periodId = periodService.addPeriod( getPeriodObj( period ) );
- Period outerPeriod = periodService.getPeriod( periodId );
+ Period outerPeriod = getPeriodObj( period );
// maps for translating identifiers
Map<String, Integer> dataelementMap = null;
@@ -259,12 +291,20 @@
do
{
// look for a DataValue
- reader.moveToStartElement( DataValueSet.DATAVALUE );
+ try
+ {
+ reader.moveToStartElement( DataValueSet.DATAVALUE );
+ } catch ( java.util.NoSuchElementException ex )
+ {
+ break;
+ }
+
if ( !reader.isStartElement( DataValueSet.DATAVALUE ) )
{
// we have to reach here eventually
break;
}
+
log.debug( "Reading Datavalue" );
String dataElementId = reader.getAttributeValue(
@@ -285,10 +325,17 @@
dv.setStoredBy( user );
dv.setTimestamp( timestamp );
- // if no outer orgunit defined, use thae map
+ // if no outer orgunit defined, use the map
if ( outerOrgunit == null )
{
+ Integer id = orgunitMap.get( innerOrgUnitId );
+ if ( id == null )
+ {
+ log.info( "Unknown orgunit: " + innerOrgUnitId + " Rejecting value");
+ continue;
+ }
dv.getSource().setId( orgunitMap.get( innerOrgUnitId ) );
+
} else
{
dv.getSource().setId( outerOrgunitId );
@@ -296,11 +343,15 @@
dv.getDataElement().setId( dataelementMap.get( dataElementId ) );
- batchHandler.addObject( dv );
+ importer.importObject(dv,params);
++countDataValues;
++cumulativeDataValueCounter;
+ if (countDataValues % DISPLAYCOUNT == 0) {
+ state.setMessage( String.format(COUNTER,cumulativeDataValueCounter));
+ }
+
log.debug( cumulativeDataValueCounter + " DataValues read" );
} while ( true ); // DataValues loop
@@ -310,17 +361,17 @@
} while ( true ); // DataValueSets loop
+ log.info( SUCCESS );
+ state.setMessage( SUCCESS );
+
} catch ( ImportException ex )
{
- log.warn( ex.toString());
+ log.warn( ex.toString() );
state.setMessage( ex.toString() );
} finally
{
batchHandler.flush();
}
-
- log.info( SUCCESS );
- state.setMessage( SUCCESS );
}
private Period getPeriodObj( String period )
@@ -339,11 +390,25 @@
try
{
periodObj = pt.createPeriod( period );
+
} catch ( Exception e )
{
throw new ImportException( String.format( INVALID_PERIOD, period ) );
}
+ Period storedPeriod = periodService.getPeriod( periodObj.getStartDate(), periodObj.getEndDate(), pt );
+
+ if ( storedPeriod == null )
+ {
+ int periodId = periodService.addPeriod( periodObj );
+
+ periodObj.setId( periodId );
+
+ } else
+ {
+ periodObj = storedPeriod;
+ }
+
return periodObj;
}
@@ -403,6 +468,7 @@
default:
throw new IllegalArgumentException( "Can't map with :" + idScheme );
}
+ log.debug( result.size() + " orgunits in map" );
return result;
}
=== modified file 'dhis-2/dhis-services/dhis-service-importexport/src/main/resources/META-INF/dhis/beans.xml'
--- dhis-2/dhis-services/dhis-service-importexport/src/main/resources/META-INF/dhis/beans.xml 2011-09-10 08:40:27 +0000
+++ dhis-2/dhis-services/dhis-service-importexport/src/main/resources/META-INF/dhis/beans.xml 2011-09-11 16:32:01 +0000
@@ -224,6 +224,8 @@
<property name="organisationUnitService" ref="org.hisp.dhis.organisationunit.OrganisationUnitService" />
<property name="periodService" ref="org.hisp.dhis.period.PeriodService" />
<property name="batchHandlerFactory" ref="batchHandlerFactory" />
+ <property name="currentUserService" ref="org.hisp.dhis.user.CurrentUserService" />
+ <property name="aggregatedDataValueService" ref="org.hisp.dhis.aggregation.AggregatedDataValueService" />
</bean>
<bean id="org.hisp.dhis.importexport.ImportService" class="org.hisp.dhis.importexport.DefaultImportService">
