dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #20242
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 9162: Initial work on new data analytics engine
------------------------------------------------------------
revno: 9162
committer: Lars Helge Øverland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Mon 2012-12-03 22:04:39 +0100
message:
Initial work on new data analytics engine
added:
dhis-2/dhis-services/dhis-service-analytics/
dhis-2/dhis-services/dhis-service-analytics/pom.xml
dhis-2/dhis-services/dhis-service-analytics/src/
dhis-2/dhis-services/dhis-service-analytics/src/main/
dhis-2/dhis-services/dhis-service-analytics/src/main/java/
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsManager.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsService.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableService.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java
dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java
dhis-2/dhis-services/dhis-service-analytics/src/main/resources/
dhis-2/dhis-services/dhis-service-analytics/src/main/resources/META-INF/
dhis-2/dhis-services/dhis-service-analytics/src/main/resources/META-INF/dhis/
dhis-2/dhis-services/dhis-service-analytics/src/main/resources/META-INF/dhis/beans.xml
modified:
dhis-2/dhis-services/pom.xml
dhis-2/pom.xml
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== added directory 'dhis-2/dhis-services/dhis-service-analytics'
=== added file 'dhis-2/dhis-services/dhis-service-analytics/pom.xml'
--- dhis-2/dhis-services/dhis-service-analytics/pom.xml 1970-01-01 00:00:00 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/pom.xml 2012-12-03 21:04:39 +0000
@@ -0,0 +1,65 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.hisp.dhis</groupId>
+ <artifactId>dhis-services</artifactId>
+ <version>2.11-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>dhis-service-analytics</artifactId>
+ <packaging>jar</packaging>
+ <name>DHIS Analytics</name>
+
+ <dependencies>
+
+ <!-- DHIS -->
+
+ <dependency>
+ <groupId>org.hisp.dhis</groupId>
+ <artifactId>dhis-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.hisp.dhis</groupId>
+ <artifactId>dhis-service-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.hisp.dhis</groupId>
+ <artifactId>dhis-service-administration</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.hisp.dhis</groupId>
+ <artifactId>dhis-support-system</artifactId>
+ </dependency>
+
+ <!-- Other -->
+
+ <dependency>
+ <groupId>jep</groupId>
+ <artifactId>jep</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.amplecode</groupId>
+ <artifactId>quick</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.whalin</groupId>
+ <artifactId>Memcached-Java-Client</artifactId>
+ <version>3.0.1</version>
+ </dependency>
+
+ </dependencies>
+ <properties>
+ <rootDir>../../</rootDir>
+ </properties>
+</project>
=== added directory 'dhis-2/dhis-services/dhis-service-analytics/src'
=== added directory 'dhis-2/dhis-services/dhis-service-analytics/src/main'
=== added directory 'dhis-2/dhis-services/dhis-service-analytics/src/main/java'
=== added directory 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org'
=== added directory 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp'
=== added directory 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis'
=== added directory 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics'
=== added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsManager.java 1970-01-01 00:00:00 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsManager.java 2012-12-03 21:04:39 +0000
@@ -0,0 +1,40 @@
+package org.hisp.dhis.analytics;
+
+/*
+ * Copyright (c) 2004-2012, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import org.hisp.dhis.aggregation.AggregatedDataValue;
+
+public interface AnalyticsManager
+{
+ Future<List<AggregatedDataValue>> getAggregatedDataValueTotals( Collection<AggregatedDataValue> values, Collection<Integer> dataElementIds,
+ Collection<String> periodIds, Collection<Integer> organisationUnitIds );
+}
=== added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsService.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsService.java 1970-01-01 00:00:00 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsService.java 2012-12-03 21:04:39 +0000
@@ -0,0 +1,39 @@
+package org.hisp.dhis.analytics;
+
+/*
+ * Copyright (c) 2004-2012, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+import java.util.Collection;
+import java.util.List;
+
+import org.hisp.dhis.aggregation.AggregatedDataValue;
+
+public interface AnalyticsService
+{
+ List<AggregatedDataValue> getAggregatedDataValueTotals( Collection<Integer> dataElementIds,
+ Collection<String> periodIds, Collection<Integer> organisationUnitIds ) throws Exception;
+}
=== added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java 1970-01-01 00:00:00 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java 2012-12-03 21:04:39 +0000
@@ -0,0 +1,54 @@
+package org.hisp.dhis.analytics;
+
+/*
+ * Copyright (c) 2004-2012, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+import java.util.List;
+import java.util.concurrent.Future;
+
+public interface AnalyticsTableManager
+{
+ void dropTable();
+
+ void createTable();
+
+ void populateTable();
+
+ Future<?> createIndexesAsync( List<String> columns );
+
+ /**
+ * Returns a list of string arrays in where the first index holds the database
+ * column name, the second index holds the database column type and the third
+ * column holds a table alias.
+ */
+ List<String[]> getDimensionColumns();
+
+ /**
+ * Returns a list of database column names.
+ */
+ List<String> getDimensionColumnNames();
+}
=== added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableService.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableService.java 1970-01-01 00:00:00 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableService.java 2012-12-03 21:04:39 +0000
@@ -0,0 +1,35 @@
+package org.hisp.dhis.analytics;
+
+/*
+ * Copyright (c) 2004-2012, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+import java.util.concurrent.Future;
+
+public interface AnalyticsTableService
+{
+ Future<?> update();
+}
=== added directory 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data'
=== added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java 1970-01-01 00:00:00 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java 2012-12-03 21:04:39 +0000
@@ -0,0 +1,73 @@
+package org.hisp.dhis.analytics.data;
+
+/*
+ * Copyright (c) 2004-2012, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import org.hisp.dhis.aggregation.AggregatedDataValue;
+import org.hisp.dhis.analytics.AnalyticsManager;
+import org.hisp.dhis.analytics.AnalyticsService;
+import org.hisp.dhis.system.util.PaginatedList;
+import org.hisp.dhis.system.util.Timer;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class DefaultAnalyticsService
+ implements AnalyticsService
+{
+ @Autowired
+ private AnalyticsManager analyticsManager;
+
+ public List<AggregatedDataValue> getAggregatedDataValueTotals( Collection<Integer> dataElementIds,
+ Collection<String> periodIds, Collection<Integer> organisationUnitIds ) throws Exception
+ {
+ Timer t = new Timer().start();
+
+ List<List<Integer>> dePages = new PaginatedList<Integer>( dataElementIds ).setNumberOfPages( 4 ).getPages();
+
+ List<Future<List<AggregatedDataValue>>> futures = new ArrayList<Future<List<AggregatedDataValue>>>();
+
+ List<AggregatedDataValue> values = new ArrayList<AggregatedDataValue>();
+
+ for ( List<Integer> dePage : dePages )
+ {
+ futures.add( analyticsManager.getAggregatedDataValueTotals( values, dePage, periodIds, organisationUnitIds ) );
+ }
+
+ for ( Future<List<AggregatedDataValue>> future : futures )
+ {
+ values.addAll( future.get() );
+ }
+
+ t.getTime( "Got aggregated values" );
+
+ return values;
+ }
+}
=== added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java 1970-01-01 00:00:00 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java 2012-12-03 21:04:39 +0000
@@ -0,0 +1,102 @@
+package org.hisp.dhis.analytics.data;
+
+/*
+ * Copyright (c) 2004-2012, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+import static org.hisp.dhis.system.util.TextUtils.getCommaDelimitedString;
+import static org.hisp.dhis.system.util.TextUtils.getQuotedCommaDelimitedString;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.hisp.dhis.aggregation.AggregatedDataValue;
+import org.hisp.dhis.analytics.AnalyticsManager;
+import org.hisp.dhis.expression.ExpressionService;
+import org.hisp.dhis.organisationunit.OrganisationUnitService;
+import org.hisp.dhis.period.PeriodService;
+import org.hisp.dhis.period.PeriodType;
+import org.hisp.dhis.system.objectmapper.AggregatedDataValueRowMapper;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.scheduling.annotation.AsyncResult;
+
+/**
+ * This class is responsible for producing aggregated data values. It reads data
+ * from the analytics table. Organisation units provided as arguments must be on
+ * the same level in the hierarchy.
+ *
+ * @author Lars Helge Overland
+ */
+public class JdbcAnalyticsManager
+ implements AnalyticsManager
+{
+ private static final Log log = LogFactory.getLog( JdbcAnalyticsManager.class );
+
+ @Autowired
+ private JdbcTemplate jdbcTemplate;
+
+ @Autowired
+ private OrganisationUnitService organisationUnitService;
+
+ @Autowired
+ private PeriodService periodService;
+
+ @Autowired
+ private ExpressionService expressionService;
+
+ // -------------------------------------------------------------------------
+ // Implementation
+ // -------------------------------------------------------------------------
+
+ //TODO period aggregation for multiple period types
+ //TODO hierarchy aggregation for org units at multiple levels
+ //TODO indicator aggregation
+
+ @Async
+ public Future<List<AggregatedDataValue>> getAggregatedDataValueTotals( Collection<AggregatedDataValue> values, Collection<Integer> dataElementIds,
+ Collection<String> periodIds, Collection<Integer> organisationUnitIds )
+ {
+ int level = organisationUnitService.getLevelOfOrganisationUnit( organisationUnitIds.iterator().next() );
+ String periodType = PeriodType.getPeriodTypeFromIsoString( periodIds.iterator().next() ).getName().toLowerCase();
+
+ final String sql =
+ "SELECT dataelementid, 0 as categoryoptioncomboid, periodid, idlevel" + level + " as organisationunitid, SUM(value) as value " +
+ "FROM analytics " +
+ "WHERE dataelementid IN ( " + getCommaDelimitedString( dataElementIds ) + " ) " +
+ "AND " + periodType + " IN ( " + getQuotedCommaDelimitedString( periodIds ) + " ) " +
+ "AND idlevel" + level + " IN ( " + getCommaDelimitedString( organisationUnitIds ) + " ) " +
+ "GROUP BY dataelementid, periodid, idlevel" + level;
+
+ log.info( sql );
+
+ return new AsyncResult<List<AggregatedDataValue>>( jdbcTemplate.query( sql, new AggregatedDataValueRowMapper() ) );
+ }
+}
=== added directory 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table'
=== added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java 1970-01-01 00:00:00 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java 2012-12-03 21:04:39 +0000
@@ -0,0 +1,97 @@
+package org.hisp.dhis.analytics.table;
+
+/*
+ * Copyright (c) 2004-2012, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.hisp.dhis.analytics.AnalyticsTableManager;
+import org.hisp.dhis.analytics.AnalyticsTableService;
+import org.hisp.dhis.system.util.ConcurrentUtils;
+import org.hisp.dhis.system.util.PaginatedList;
+import org.hisp.dhis.system.util.SystemUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.annotation.Async;
+
+public class DefaultAnalyticsTableService
+ implements AnalyticsTableService
+{
+ private static final Log log = LogFactory.getLog( DefaultAnalyticsTableService.class );
+
+ @Autowired
+ private AnalyticsTableManager tableManager;
+
+ // -------------------------------------------------------------------------
+ // Implementation
+ // -------------------------------------------------------------------------
+
+ //TODO generateOrganisationUnitStructures
+ //TODO generateOrganisationUnitGroupSetTable
+ //TODO generatePeriodStructure
+
+ @Async
+ public Future<?> update()
+ {
+ log.info( "Starting update..." );
+
+ tableManager.dropTable();
+ tableManager.createTable();
+ tableManager.populateTable();
+
+ createIndexes();
+
+ log.info( "Update done" );
+
+ return null;
+ }
+
+ // -------------------------------------------------------------------------
+ // Supportive methods
+ // -------------------------------------------------------------------------
+
+ private void createIndexes()
+ {
+ int pages = Math.max( ( SystemUtils.getCpuCores() - 1 ), 1 );
+
+ log.info( "No of pages: " + pages );
+
+ List<Future<?>> futures = new ArrayList<Future<?>>();
+
+ List<List<String>> columnPages = new PaginatedList<String>( tableManager.getDimensionColumnNames() ).setNumberOfPages( pages ).getPages();
+
+ for ( List<String> columnPage : columnPages )
+ {
+ futures.add( tableManager.createIndexesAsync( columnPage ) );
+ }
+
+ ConcurrentUtils.waitForCompletion( futures );
+ }
+}
=== added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java 1970-01-01 00:00:00 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java 2012-12-03 21:04:39 +0000
@@ -0,0 +1,244 @@
+package org.hisp.dhis.analytics.table;
+
+/*
+ * Copyright (c) 2004-2012, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.hisp.dhis.analytics.AnalyticsTableManager;
+import org.hisp.dhis.organisationunit.OrganisationUnitGroupService;
+import org.hisp.dhis.organisationunit.OrganisationUnitGroupSet;
+import org.hisp.dhis.organisationunit.OrganisationUnitLevel;
+import org.hisp.dhis.organisationunit.OrganisationUnitService;
+import org.hisp.dhis.period.PeriodType;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.jdbc.BadSqlGrammarException;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.scheduling.annotation.Async;
+
+/**
+ * This class manages the analytics table. The analytics table is a denormalized
+ * table designed for analysis which contains raw data values. It has columns for
+ * each organisation unit group set and organisation unit level. Also, columns
+ * for dataelementid, periodid, organisationunitid, categoryoptioncomboid, value.
+ *
+ * The data records in this table are not aggregated. Typically, queries will
+ * aggregate in organisation unit hierarchy dimension, in the period/time dimension,
+ * and the category dimensions, as well as org unit group set dimensions.
+ *
+ * @author Lars Helge Overland
+ */
+public class JdbcAnalyticsTableManager
+ implements AnalyticsTableManager
+{
+ private static final Log log = LogFactory.getLog( JdbcAnalyticsTableManager.class );
+
+ public static final String PREFIX_ORGUNITGROUPSET = "ougs_";
+ public static final String PREFIX_ORGUNITLEVEL = "idlevel";
+ public static final String PREFIX_INDEX = "index_";
+ public static final String TABLE_NAME = "analytics";
+
+ @Autowired
+ private OrganisationUnitService organisationUnitService;
+
+ @Autowired
+ private OrganisationUnitGroupService organisationUnitGroupService;
+
+ @Autowired
+ private JdbcTemplate jdbcTemplate;
+
+ // -------------------------------------------------------------------------
+ // Implementation
+ // -------------------------------------------------------------------------
+
+ //TODO all data types
+ //TODO create temp table then swap
+ //TODO shard on data year
+ //TODO average aggregation operator data, pre-aggregate in time dimension, not in org unit dimension
+
+ public void dropTable()
+ {
+ final String sql = "drop table " + TABLE_NAME;
+
+ executeSilently( sql );
+
+ log.info( "Dropped table: " + TABLE_NAME );
+ }
+
+ public void createTable()
+ {
+ String sql = "create table " + TABLE_NAME + " (";
+
+ for ( String[] col : getDimensionColumns() )
+ {
+ sql += col[0] + " " + col[1] + ",";
+ }
+
+ sql += "value double precision)";
+
+ log.info( "Create SQL: " + sql );
+
+ executeSilently( sql );
+
+ log.info( "Created table: " + TABLE_NAME );
+ }
+
+ @Async
+ public Future<?> createIndexesAsync( List<String> columns )
+ {
+ for ( String column : columns )
+ {
+ final String sql = "create index " + PREFIX_INDEX +
+ column + " on " + TABLE_NAME + " (" + column + ")";
+
+ executeSilently( sql );
+
+ log.info( "Created index on column: " + column );
+ }
+
+ log.info( "Indexes created" );
+
+ return null;
+ }
+
+ public void populateTable()
+ {
+ String insert = "insert into analytics (";
+
+ for ( String[] col : getDimensionColumns() )
+ {
+ insert += col[0] + ",";
+ }
+
+ insert += "value) ";
+
+ String select = "select ";
+
+ for ( String[] col : getDimensionColumns() )
+ {
+ select += col[2] + col[0] + ",";
+ }
+
+ select = select.replace( "organisationunitid", "sourceid" ); // Legacy fix
+
+ select +=
+ "cast(dv.value as double precision) " +
+ "from datavalue dv " +
+ "left join _organisationunitgroupsetstructure ougs on dv.sourceid=ougs.organisationunitid " +
+ "left join _orgunitstructure ous on dv.sourceid=ous.organisationunitid " +
+ "left join _period_no_disaggregation_structure ps on dv.periodid=ps.periodid " +
+ "left join dataelement de on dv.dataelementid=de.dataelementid " +
+ "left join period pe on dv.periodid=pe.periodid " +
+ "where de.valuetype='int' and pe.startdate >= '2011-10-01'";
+
+ final String sql = insert + select;
+
+ log.info( "Populate SQL: "+ sql );
+
+ jdbcTemplate.execute( sql );
+
+ log.info( "Populated analytics table" );
+ }
+
+ public List<String[]> getDimensionColumns()
+ {
+ List<String[]> columns = new ArrayList<String[]>();
+
+ Collection<OrganisationUnitGroupSet> orgUnitGroupSets =
+ organisationUnitGroupService.getCompulsoryOrganisationUnitGroupSets();
+
+ Collection<OrganisationUnitLevel> levels =
+ organisationUnitService.getOrganisationUnitLevels();
+
+ for ( OrganisationUnitGroupSet groupSet : orgUnitGroupSets )
+ {
+ String[] col = { PREFIX_ORGUNITGROUPSET + groupSet.getUid(), "integer", "ougs." };
+ columns.add( col );
+ }
+
+ for ( OrganisationUnitLevel level : levels )
+ {
+ String[] col = { PREFIX_ORGUNITLEVEL + level.getLevel(), "integer", "ous." };
+ columns.add( col );
+ }
+
+ for ( PeriodType periodType : PeriodType.getAvailablePeriodTypes().subList( 0, 7 ) )
+ {
+ String[] col = { periodType.getName().toLowerCase(), "character varying(10)", "ps." };
+ columns.add( col );
+ }
+
+ String[] de = { "dataelementid", "integer not null", "dv." };
+ String[] pe = { "periodid", "integer not null", "dv." };
+ String[] ou = { "organisationunitid", "integer not null", "dv." };
+ String[] co = { "categoryoptioncomboid", "integer not null", "dv." };
+
+ columns.addAll( Arrays.asList( de, pe, ou, co ) );
+
+ return columns;
+ }
+
+ public List<String> getDimensionColumnNames()
+ {
+ List<String[]> columns = getDimensionColumns();
+
+ List<String> columnNames = new ArrayList<String>();
+
+ for ( String[] column : columns )
+ {
+ columnNames.add( column[0] );
+ }
+
+ return columnNames;
+ }
+
+ // -------------------------------------------------------------------------
+ // Supportive methods
+ // -------------------------------------------------------------------------
+
+ /**
+ * Executes a SQL statement. Ignores existing tables/indexes when attempting
+ * to create new.
+ */
+ private void executeSilently( String sql )
+ {
+ try
+ {
+ jdbcTemplate.execute( sql );
+ }
+ catch ( BadSqlGrammarException ex )
+ {
+ log.warn( ex.getMessage() );
+ }
+ }
+}
=== added directory 'dhis-2/dhis-services/dhis-service-analytics/src/main/resources'
=== added directory 'dhis-2/dhis-services/dhis-service-analytics/src/main/resources/META-INF'
=== added directory 'dhis-2/dhis-services/dhis-service-analytics/src/main/resources/META-INF/dhis'
=== added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/resources/META-INF/dhis/beans.xml'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/resources/META-INF/dhis/beans.xml 1970-01-01 00:00:00 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/resources/META-INF/dhis/beans.xml 2012-12-03 21:04:39 +0000
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns="http://www.springframework.org/schema/beans"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xmlns:aop="http://www.springframework.org/schema/aop"; xmlns:sec="http://www.springframework.org/schema/security";
+ xsi:schemaLocation="
+http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
+http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-3.1.xsd
+http://www.springframework.org/schema/security http://www.springframework.org/schema/security/spring-security-3.1.xsd";>
+
+ <bean id="org.hisp.dhis.analytics.AnalyticsTableManager" class="org.hisp.dhis.analytics.table.JdbcAnalyticsTableManager" />
+
+ <bean id="org.hisp.dhis.analytics.AnalyticsTableService" class="org.hisp.dhis.analytics.table.DefaultAnalyticsTableService" />
+
+ <bean id="org.hisp.dhis.analytics.AnalyticsManager" class="org.hisp.dhis.analytics.data.JdbcAnalyticsManager" />
+
+ <bean id="org.hisp.dhis.analytics.AnalyticsService" class="org.hisp.dhis.analytics.data.DefaultAnalyticsService" />
+
+</beans>
=== modified file 'dhis-2/dhis-services/pom.xml'
--- dhis-2/dhis-services/pom.xml 2012-11-20 13:38:49 +0000
+++ dhis-2/dhis-services/pom.xml 2012-12-03 21:04:39 +0000
@@ -22,6 +22,7 @@
<module>dhis-service-reporting</module>
<module>dhis-service-mapgeneration</module>
<module>dhis-service-aggregationengine-default</module>
+ <module>dhis-service-analytics</module>
<module>dhis-service-sms</module>
<module>dhis-service-patient</module>
<module>dhis-service-mobile</module>
=== modified file 'dhis-2/pom.xml'
--- dhis-2/pom.xml 2012-11-29 16:21:29 +0000
+++ dhis-2/pom.xml 2012-12-03 21:04:39 +0000
@@ -285,6 +285,11 @@
</dependency>
<dependency>
<groupId>org.hisp.dhis</groupId>
+ <artifactId>dhis-service-analytics</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.hisp.dhis</groupId>
<artifactId>dhis-support-jdbc</artifactId>
<version>${project.version}</version>
</dependency>
