dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #31529
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 16077: Impl first cut of function for synchronizing data values from one instance of DHIS to another onl...
------------------------------------------------------------
revno: 16077
committer: Lars Helge Overland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Thu 2014-07-10 18:06:48 +0200
message:
Impl first cut of function for synchronizing data values from one instance of DHIS to another online instance of DHIS. Uses streaming POST request for data value set through spring rest template. Using basic auth. Stores time of last synch success. Converts the response into an ImportSummary and checks for status of synch process.
added:
dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/utils/ImportSummaryResponseExtractor.java
modified:
dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DataValueSetService.java
dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DataValueSetStore.java
dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DefaultDataValueSetService.java
dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/SpringDataValueSetStore.java
dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/importsummary/ImportConflict.java
dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/synch/DefaultSynchronizationManager.java
dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/synch/SynchronizationManager.java
dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/SynchronizationController.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DataValueSetService.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DataValueSetService.java 2014-06-12 16:30:25 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DataValueSetService.java 2014-07-10 16:06:48 +0000
@@ -52,6 +52,8 @@
void writeDataValueSetJson( Set<String> dataSet, Date startDate, Date endDate, Set<String> ous, OutputStream outputStream );
+ void writeDataValueSetJson( Date lastUpdated, OutputStream outputStream );
+
void writeDataValueSetCsv( Set<String> dataSets, Date startDate, Date endDate, Set<String> orgUnits, Writer writer );
RootNode getDataValueSetTemplate( DataSet dataSet, Period period, List<String> orgUnits,
=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DataValueSetStore.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DataValueSetStore.java 2014-03-18 08:10:10 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DataValueSetStore.java 2014-07-10 16:06:48 +0000
@@ -38,13 +38,18 @@
import org.hisp.dhis.organisationunit.OrganisationUnit;
import org.hisp.dhis.period.Period;
+/**
+ * @author Lars Helge Overland
+ */
public interface DataValueSetStore
{
public void writeDataValueSetXml( DataSet dataSet, Date completeDate, Period period, OrganisationUnit orgUnit,
Set<DataElement> dataElements, Set<Period> periods, Set<OrganisationUnit> orgUnits, OutputStream out );
- public void writeDataValueSetCsv( Set<DataElement> dataElements,
- Set<Period> periods, Set<OrganisationUnit> orgUnits, Writer writer );
+ public void writeDataValueSetCsv( Set<DataElement> dataElements, Set<Period> periods,
+ Set<OrganisationUnit> orgUnits, Writer writer );
+
+ void writeDataValueSetJson( Date lastUpdated, OutputStream outputStream );
public void writeDataValueSetJson( DataSet dataSet, Date completeDate, Period period, OrganisationUnit orgUnit,
Set<DataElement> dataElements, Set<Period> periods, Set<OrganisationUnit> orgUnits, OutputStream out );
=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DefaultDataValueSetService.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DefaultDataValueSetService.java 2014-07-10 12:23:09 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DefaultDataValueSetService.java 2014-07-10 16:06:48 +0000
@@ -217,6 +217,12 @@
{
dataValueSetStore.writeDataValueSetJson( null, null, null, null, getDataElements( dataSets ), getPeriods( startDate, endDate ), getOrgUnits( orgUnits ), outputStream );
}
+
+ @Override
+ public void writeDataValueSetJson( Date lastUpdated, OutputStream outputStream )
+ {
+ dataValueSetStore.writeDataValueSetJson( lastUpdated, outputStream );
+ }
@Override
public void writeDataValueSetCsv( Set<String> dataSets, Date startDate, Date endDate, Set<String> orgUnits, Writer writer )
=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/SpringDataValueSetStore.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/SpringDataValueSetStore.java 2014-07-10 14:16:24 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/SpringDataValueSetStore.java 2014-07-10 16:06:48 +0000
@@ -73,8 +73,10 @@
Set<DataElement> dataElements, Set<Period> periods, Set<OrganisationUnit> orgUnits, OutputStream out )
{
DataValueSet dataValueSet = new StreamingDataValueSet( XMLFactory.getXMLWriter( out ) );
-
- writeDataValueSet( dataSet, completeDate, period, orgUnit, dataElements, periods, orgUnits, dataValueSet );
+
+ SqlRowSet rowSet = jdbcTemplate.queryForRowSet( getDataValueSql( dataElements, periods, orgUnits ) );
+
+ writeDataValueSet( rowSet, dataSet, completeDate, period, orgUnit, dataValueSet );
StreamUtils.closeOutputStream( out );
}
@@ -84,29 +86,50 @@
Set<DataElement> dataElements, Set<Period> periods, Set<OrganisationUnit> orgUnits, OutputStream outputStream )
{
DataValueSet dataValueSet = new StreamingJsonDataValueSet( outputStream );
-
- writeDataValueSet( dataSet, completeDate, period, orgUnit, dataElements, periods, orgUnits, dataValueSet );
+
+ SqlRowSet rowSet = jdbcTemplate.queryForRowSet( getDataValueSql( dataElements, periods, orgUnits ) );
+
+ writeDataValueSet( rowSet, dataSet, completeDate, period, orgUnit, dataValueSet );
StreamUtils.closeOutputStream( outputStream );
}
@Override
+ public void writeDataValueSetJson( Date lastUpdated, OutputStream outputStream )
+ {
+ DataValueSet dataValueSet = new StreamingJsonDataValueSet( outputStream );
+
+ final String sql =
+ "select de.uid as deuid, pe.startdate, pt.name, ou.uid as ouuid, coc.uid as cocuid, dv.value, dv.storedby, dv.created, dv.lastupdated, dv.comment, dv.followup " +
+ "from datavalue dv " +
+ "join dataelement de on (dv.dataelementid=de.dataelementid) " +
+ "join period pe on (dv.periodid=pe.periodid) " +
+ "join periodtype pt on (pe.periodtypeid=pt.periodtypeid) " +
+ "join organisationunit ou on (dv.sourceid=ou.organisationunitid) " +
+ "join categoryoptioncombo coc on (dv.categoryoptioncomboid=coc.categoryoptioncomboid) " +
+ "where dv.lastupdated >= '" + DateUtils.getLongDateString( lastUpdated ) + "'";
+
+ SqlRowSet rowSet = jdbcTemplate.queryForRowSet( sql );
+
+ writeDataValueSet( rowSet, null, null, null, null, dataValueSet );
+ }
+
+ @Override
public void writeDataValueSetCsv( Set<DataElement> dataElements, Set<Period> periods, Set<OrganisationUnit> orgUnits, Writer writer )
{
DataValueSet dataValueSet = new StreamingCsvDataValueSet( new CsvWriter( writer, CSV_DELIM ) );
-
- writeDataValueSet( null, null, null, null, dataElements, periods, orgUnits, dataValueSet );
+
+ SqlRowSet rowSet = jdbcTemplate.queryForRowSet( getDataValueSql( dataElements, periods, orgUnits ) );
+
+ writeDataValueSet( rowSet, null, null, null, null, dataValueSet );
}
-
+
//--------------------------------------------------------------------------
// Supportive methods
//--------------------------------------------------------------------------
- private void writeDataValueSet( DataSet dataSet, Date completeDate, Period period, OrganisationUnit orgUnit,
- Set<DataElement> dataElements, Set<Period> periods, Set<OrganisationUnit> orgUnits, DataValueSet dataValueSet )
+ private void writeDataValueSet( SqlRowSet rowSet, DataSet dataSet, Date completeDate, Period period, OrganisationUnit orgUnit, DataValueSet dataValueSet )
{
- SqlRowSet rowSet = jdbcTemplate.queryForRowSet( getDataValueSql( dataElements, periods, orgUnits ) );
-
dataValueSet.setDataSet( dataSet != null ? dataSet.getUid() : null );
dataValueSet.setCompleteDate( getMediumDateString( completeDate ) );
dataValueSet.setPeriod( period != null ? period.getIsoDate() : null );
=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/importsummary/ImportConflict.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/importsummary/ImportConflict.java 2014-03-18 08:10:10 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/importsummary/ImportConflict.java 2014-07-10 16:06:48 +0000
@@ -40,6 +40,10 @@
private String value;
+ public ImportConflict()
+ {
+ }
+
public ImportConflict( String object, String value )
{
this.object = object;
=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/synch/DefaultSynchronizationManager.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/synch/DefaultSynchronizationManager.java 2014-07-07 11:22:34 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/synch/DefaultSynchronizationManager.java 2014-07-10 16:06:48 +0000
@@ -30,6 +30,7 @@
import static org.apache.commons.lang.StringUtils.trimToNull;
+import java.io.IOException;
import java.util.Date;
import org.apache.commons.logging.Log;
@@ -38,6 +39,9 @@
import org.hisp.dhis.configuration.ConfigurationService;
import org.hisp.dhis.datavalue.DataValueService;
import org.hisp.dhis.dxf2.datavalueset.DataValueSetService;
+import org.hisp.dhis.dxf2.importsummary.ImportStatus;
+import org.hisp.dhis.dxf2.importsummary.ImportSummary;
+import org.hisp.dhis.dxf2.utils.ImportSummaryResponseExtractor;
import org.hisp.dhis.setting.SystemSettingManager;
import org.hisp.dhis.system.util.CodecUtils;
import org.springframework.beans.factory.annotation.Autowired;
@@ -45,12 +49,16 @@
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
+import org.springframework.http.client.ClientHttpRequest;
import org.springframework.scheduling.TaskScheduler;
import org.springframework.scheduling.support.CronTrigger;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.HttpServerErrorException;
+import org.springframework.web.client.RequestCallback;
import org.springframework.web.client.ResourceAccessException;
+import org.springframework.web.client.ResponseExtractor;
import org.springframework.web.client.RestTemplate;
/**
@@ -168,31 +176,62 @@
return false;
}
- public boolean executeDataSynch()
+ public ImportSummary executeDataSynch()
{
AvailabilityStatus availability = isRemoteServerAvailable();
if ( !availability.isAvailable() )
{
log.info( "Aborting synch, server not available" );
- return false;
+ return null;
}
- Date date = getLastSynchSuccess();
+ // ---------------------------------------------------------------------
+ // Set time for last success to start of process to make data saved
+ // subsequently part of next synch process without being ignored
+ // ---------------------------------------------------------------------
+
+ final Date time = getLastSynchSuccess();
- int lastUpdatedCount = dataValueService.getDataValueCountLastUpdatedAfter( date );
+ int lastUpdatedCount = dataValueService.getDataValueCountLastUpdatedAfter( time );
if ( lastUpdatedCount == 0 )
{
log.info( "Aborting synch, no new or updated data values" );
- return false;
+ return null;
}
-
- // Synch
-
- setLastSynchSuccess();
-
- return true;
+
+ final Configuration config = configurationService.getConfiguration();
+
+ String url = config.getRemoteServerUrl() + "/api/dataValueSets";
+
+ log.info( "Remote server POST URL: " + url );
+
+ final RequestCallback requestCallback = new RequestCallback() {
+
+ public void doWithRequest( ClientHttpRequest request ) throws IOException
+ {
+ request.getHeaders().setContentType( MediaType.APPLICATION_JSON );
+ request.getHeaders().add( HEADER_AUTHORIZATION, CodecUtils.getBasicAuthString( config.getRemoteServerUsername(), config.getRemoteServerPassword() ) );
+ dataValueSetService.writeDataValueSetJson( time, request.getBody() );
+ }
+ };
+
+ ResponseExtractor<ImportSummary> responseExtractor = new ImportSummaryResponseExtractor();
+
+ ImportSummary summary = restTemplate.execute( url, HttpMethod.POST, requestCallback, responseExtractor );
+
+ log.info( summary );
+
+ if ( summary != null && ImportStatus.SUCCESS.equals( summary.getStatus() ) )
+ {
+ setLastSynchSuccess( time );
+ log.info( "Synch successful, setting last success time: " + time );
+ }
+
+ return summary;
+
+ //TODO paging of data values
}
// -------------------------------------------------------------------------
@@ -213,9 +252,9 @@
/**
* Sets the time of the last successful synchronization operation.
*/
- private void setLastSynchSuccess()
+ private void setLastSynchSuccess( Date time )
{
- systemSettingManager.saveSystemSetting( KEY_LAST_SUCCESSFUL_SYNC, new Date() );
+ systemSettingManager.saveSystemSetting( KEY_LAST_SUCCESSFUL_SYNC, time );
}
/**
=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/synch/SynchronizationManager.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/synch/SynchronizationManager.java 2014-07-07 11:00:35 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/synch/SynchronizationManager.java 2014-07-10 16:06:48 +0000
@@ -28,6 +28,8 @@
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
+import org.hisp.dhis.dxf2.importsummary.ImportSummary;
+
/**
* @author Lars Helge Overland
*/
@@ -35,7 +37,7 @@
{
AvailabilityStatus isRemoteServerAvailable();
- boolean executeDataSynch();
+ ImportSummary executeDataSynch();
void enableDataSynch();
=== added file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/utils/ImportSummaryResponseExtractor.java'
--- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/utils/ImportSummaryResponseExtractor.java 1970-01-01 00:00:00 +0000
+++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/utils/ImportSummaryResponseExtractor.java 2014-07-10 16:06:48 +0000
@@ -0,0 +1,64 @@
+package org.hisp.dhis.dxf2.utils;
+
+/*
+ * Copyright (c) 2004-2014, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+import java.io.IOException;
+
+import org.hisp.dhis.dxf2.importsummary.ImportSummary;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.client.ClientHttpResponse;
+import org.springframework.web.client.HttpServerErrorException;
+import org.springframework.web.client.ResponseExtractor;
+
+/**
+ * Converts a response into an ImportSummary instance.
+ *
+ * @throws HttpServerErrorException if the response status code is different
+ * from 200 OK or 201 Created.
+ * @throws IOException if converting the response into an ImportSummary failed.
+ *
+ * @author Lars Helge Overland
+ */
+public class ImportSummaryResponseExtractor
+ implements ResponseExtractor<ImportSummary>
+{
+ public ImportSummary extractData( ClientHttpResponse response ) throws IOException
+ {
+ ImportSummary summary = JacksonUtils.fromJson( response.getBody(), ImportSummary.class );
+
+ HttpStatus status = response.getStatusCode();
+
+ if ( !( HttpStatus.CREATED.equals( status ) || HttpStatus.OK.equals( status ) ) )
+ {
+ throw new HttpServerErrorException( status, "Data synch failed on remote server" );
+ }
+
+ return summary;
+ }
+}
=== modified file 'dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/SynchronizationController.java'
--- dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/SynchronizationController.java 2014-07-07 10:06:36 +0000
+++ dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/SynchronizationController.java 2014-07-10 16:06:48 +0000
@@ -28,8 +28,16 @@
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
+import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_JSON;
+
+import java.io.IOException;
+
+import javax.servlet.http.HttpServletResponse;
+
+import org.hisp.dhis.dxf2.importsummary.ImportSummary;
import org.hisp.dhis.dxf2.synch.AvailabilityStatus;
import org.hisp.dhis.dxf2.synch.SynchronizationManager;
+import org.hisp.dhis.dxf2.utils.JacksonUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
@@ -52,9 +60,19 @@
@Autowired
private SynchronizationManager synchronizationManager;
+ @RequestMapping( method = RequestMethod.POST )
+ public void execute( HttpServletResponse response )
+ throws IOException
+ {
+ ImportSummary summary = synchronizationManager.executeDataSynch();
+
+ response.setContentType( CONTENT_TYPE_JSON );
+ JacksonUtils.toJson( response.getOutputStream(), summary );
+ }
+
@RequestMapping( value = "/availability", method = RequestMethod.GET, produces = "application/json" )
public @ResponseBody AvailabilityStatus isRemoteServerAvailable()
{
return synchronizationManager.isRemoteServerAvailable();
- }
+ }
}
