dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #09424
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 2499: Improved trimming of crosstab table in datamart
------------------------------------------------------------
revno: 2499
committer: Lars Helge Overland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Sun 2011-01-09 22:24:40 +0100
message:
Improved trimming of crosstab table in datamart
modified:
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/util/ParserUtil.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java 2011-01-09 13:02:36 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java 2011-01-09 21:24:40 +0000
@@ -49,7 +49,7 @@
* @param organisationUnitIds the collection of OrganisationUnit identifiers.
* @return the DataElementOperands where data exists.
*/
- boolean populateAndTrimCrossTabTable( final Collection<DataElementOperand> operands,
+ Collection<DataElementOperand> populateAndTrimCrossTabTable( final Collection<DataElementOperand> operands,
final Collection<Integer> periodIds, final Collection<Integer> organisationUnitIds, String key );
/**
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java 2011-01-09 13:02:36 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java 2011-01-09 21:24:40 +0000
@@ -87,14 +87,14 @@
// CrossTabService implementation
// -------------------------------------------------------------------------
- public boolean populateAndTrimCrossTabTable( final Collection<DataElementOperand> operands,
+ public Collection<DataElementOperand> populateAndTrimCrossTabTable( final Collection<DataElementOperand> operands,
final Collection<Integer> periodIds, final Collection<Integer> organisationUnitIds, String key )
{
Collection<DataElementOperand> operandsWithData = populateCrossTabTable( operands, periodIds, organisationUnitIds, key );
trimCrossTabTable( operandsWithData, key );
- return operandsWithData != null;
+ return operandsWithData;
}
public Collection<DataElementOperand> populateCrossTabTable( final Collection<DataElementOperand> operands,
@@ -102,7 +102,7 @@
{
if ( validate( operands, periodIds, organisationUnitIds ) )
{
- final Set<DataElementOperand> operandsWithData = new HashSet<DataElementOperand>( operands );
+ final Set<DataElementOperand> operandsWithData = new HashSet<DataElementOperand>();
final List<DataElementOperand> operandList = new ArrayList<DataElementOperand>( operands );
@@ -116,8 +116,7 @@
log.info( "Created crosstab table" );
- final BatchHandler<Object> batchHandler = batchHandlerFactory
- .createBatchHandler( GenericBatchHandler.class );
+ final BatchHandler<Object> batchHandler = batchHandlerFactory.createBatchHandler( GenericBatchHandler.class );
batchHandler.setTableName( CrossTabStore.TABLE_NAME + key );
batchHandler.init();
@@ -190,7 +189,8 @@
public void trimCrossTabTable( Collection<DataElementOperand> operands, String key )
{
// TODO use H2 in-memory table for datavaluecrosstab table ?
-
+ // TODO more compact crosstab tables by dividing up based on periodtype?
+
if ( operands != null && key != null )
{
crossTabStore.createTrimmedCrossTabTable( operands, key );
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2011-01-09 20:16:36 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2011-01-09 21:24:40 +0000
@@ -235,7 +235,7 @@
}
// ---------------------------------------------------------------------
- // Filter and get operands
+ // Get operands
// ---------------------------------------------------------------------
Collection<DataElementOperand> nonCalculatedOperands = categoryService.getOperands( nonCalculatedDataElements );
@@ -268,20 +268,27 @@
state.setMessage( "crosstabulating_data" );
- Collection<Integer> childrenIds = organisationUnitService.getOrganisationUnitHierarchy().getChildren(
- organisationUnitIds );
-
+ Collection<Integer> childrenIds = organisationUnitService.getOrganisationUnitHierarchy().getChildren( organisationUnitIds );
Collection<Integer> intersectingPeriodIds = ConversionUtils.getIdentifiers( Period.class, periodService.getIntersectionPeriods( periods ) );
- boolean valid = crossTabService.populateAndTrimCrossTabTable( allOperands, intersectingPeriodIds, childrenIds, key );
+ Collection<DataElementOperand> operandsWithData = crossTabService.populateAndTrimCrossTabTable( allOperands, intersectingPeriodIds, childrenIds, key );
- if ( !valid )
- {
- return 0;
- }
-
log.info( "Populated crosstab table: " + TimeUtils.getHMS() );
+ if ( operandsWithData == null )
+ {
+ return 0;
+ }
+
+ // ---------------------------------------------------------------------
+ // Remove operands without data
+ // ---------------------------------------------------------------------
+
+ nonCalculatedOperands.retainAll( operandsWithData );
+ indicatorOperands.retainAll( operandsWithData );
+ calculatedOperands.retainAll( operandsWithData );
+ allOperands.retainAll( operandsWithData );
+
// ---------------------------------------------------------------------
// Data element export
// ---------------------------------------------------------------------
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/util/ParserUtil.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/util/ParserUtil.java 2010-12-08 15:13:31 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/util/ParserUtil.java 2011-01-09 21:24:40 +0000
@@ -29,8 +29,6 @@
import static org.hisp.dhis.expression.Expression.SEPARATOR;
-import java.util.ArrayList;
-import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
@@ -118,27 +116,4 @@
throw new RuntimeException( "Illegal data element or category combo id", ex );
}
}
-
- /**
- * Filters the data element operands based on the value type.
- *
- * @param valueType the value type to filter.
- * @param aggregationOperator the aggregation operator to filter.
- */
- public static Collection<DataElementOperand> filterOperands( Collection<DataElementOperand> operands, String valueType,
- String aggregationOperator )
- {
- final Collection<DataElementOperand> filtered = new ArrayList<DataElementOperand>();
-
- for ( DataElementOperand operand : operands )
- {
- if ( operand.getValueType().equals( valueType )
- && operand.getAggregationOperator().equals( aggregationOperator ) )
- {
- filtered.add( operand );
- }
- }
-
- return filtered;
- }
}
