dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #08975
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 2280: Significant performance improvement for indicator datamart
------------------------------------------------------------
revno: 2280
committer: Lars Helge Overland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Sat 2010-12-04 01:03:38 +0100
message:
Significant performance improvement for indicator datamart
modified:
dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataelement/DataElementCategoryService.java
dhis-2/dhis-api/src/main/java/org/hisp/dhis/expression/ExpressionService.java
dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataelement/DefaultDataElementCategoryService.java
dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/DefaultExpressionService.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataelement/DataElementCategoryService.java'
--- dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataelement/DataElementCategoryService.java 2010-12-03 23:12:42 +0000
+++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataelement/DataElementCategoryService.java 2010-12-04 00:03:38 +0000
@@ -324,13 +324,6 @@
public Collection<DataElementOperand> populateOperands( Collection<DataElementOperand> operands );
/**
- *
- * @param dataElementIdentifiers
- * @return
- */
- Collection<DataElementOperand> getOperandsByIds( Collection<Integer> dataElementIdentifiers );
-
- /**
* Gets the Operands for the given Collection of DataElements.
*
* @param dataElements the Collection of DataElements.
=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/expression/ExpressionService.java'
--- dhis-2/dhis-api/src/main/java/org/hisp/dhis/expression/ExpressionService.java 2010-12-02 22:32:52 +0000
+++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/expression/ExpressionService.java 2010-12-04 00:03:38 +0000
@@ -31,7 +31,6 @@
import java.util.Map;
import java.util.Set;
-import org.hisp.dhis.dataelement.CalculatedDataElement;
import org.hisp.dhis.dataelement.DataElement;
import org.hisp.dhis.dataelement.DataElementOperand;
import org.hisp.dhis.period.Period;
@@ -110,14 +109,6 @@
Double getExpressionValue( Expression expression, Period period, Source source, boolean nullIfNoValues, boolean aggregate );
/**
- * Returns all DataElements associated with the CalculatedDataElement.
- *
- * @param id the CalculatedDataElement identifier.
- * @return a Set of DataElements.
- */
- Set<DataElement> getDataElementsInCalculatedDataElement( CalculatedDataElement calculatedDataElement );
-
- /**
* Returns all DataElements included in the given expression string.
*
* @param expression The expression string.
=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataelement/DefaultDataElementCategoryService.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataelement/DefaultDataElementCategoryService.java 2010-12-03 23:12:42 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataelement/DefaultDataElementCategoryService.java 2010-12-04 00:03:38 +0000
@@ -388,18 +388,11 @@
return categoryCombo.getOptionCombos().iterator().next();
}
- public Collection<DataElementOperand> getOperandsByIds( Collection<Integer> dataElementIdentifiers )
- {
- Collection<DataElement> dataElements = dataElementService.getDataElements( dataElementIdentifiers );
-
- return getOperands( dataElements );
- }
-
public Collection<DataElementOperand> populateOperands( Collection<DataElementOperand> operands )
{
for ( DataElementOperand operand : operands )
{
- DataElement dataElement = dataElementService.getDataElement( operand.getId() );
+ DataElement dataElement = dataElementService.getDataElement( operand.getDataElementId() );
DataElementCategoryOptionCombo categoryOptionCombo = getDataElementCategoryOptionCombo( operand.getOptionComboId() );
operand.updateProperties( dataElement, categoryOptionCombo );
=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/DefaultExpressionService.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/DefaultExpressionService.java 2010-12-02 22:32:52 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/DefaultExpressionService.java 2010-12-04 00:03:38 +0000
@@ -152,11 +152,6 @@
return expressionString != null ? calculateExpression( expressionString ) : null;
}
- public Set<DataElement> getDataElementsInCalculatedDataElement( CalculatedDataElement calculatedDataElement )
- {
- return calculatedDataElement != null ? getDataElementsInExpression( calculatedDataElement.getExpression().getExpression() ) : null;
- }
-
public Set<DataElement> getDataElementsInExpression( String expression )
{
Set<DataElement> dataElementsInExpression = null;
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2010-12-02 23:59:04 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2010-12-04 00:03:38 +0000
@@ -31,7 +31,6 @@
import static org.hisp.dhis.dataelement.DataElement.AGGREGATION_OPERATOR_SUM;
import static org.hisp.dhis.dataelement.DataElement.VALUE_TYPE_BOOL;
import static org.hisp.dhis.dataelement.DataElement.VALUE_TYPE_INT;
-import static org.hisp.dhis.datamart.util.ParserUtil.getDataElementIdsInExpression;
import java.util.Collection;
import java.util.HashSet;
@@ -235,6 +234,7 @@
// Filter and get operands
// ---------------------------------------------------------------------
+ /*
final Collection<Integer> nonCalculatedDataElementIds = ConversionUtils.getIdentifiers( DataElement.class, nonCalculatedDataElements );
final Set<Integer> dataElementInIndicatorIds = getDataElementIdsInIndicators( indicators );
final Set<Integer> dataElementInCalculatedDataElementIds = getDataElementIdsInCalculatedDataElements( calculatedDataElements );
@@ -252,15 +252,25 @@
.getOperandsByIds( dataElementInCalculatedDataElementIds );
final Collection<DataElementOperand> nonCalculatedDataElementOperands = categoryService
.getOperandsByIds( nonCalculatedDataElementIds );
+ */
+
+ Collection<DataElementOperand> nonCalculatedOperands = categoryService.getOperands( nonCalculatedDataElements );
+ Collection<DataElementOperand> indicatorOperands = categoryService.populateOperands( getOperandsInIndicators( indicators ) );
+ Collection<DataElementOperand> calculatedOperands = categoryService.populateOperands( getOperandsInCalculatedDataElements( calculatedDataElements ) );
+
+ Set<DataElementOperand> allOperands = new HashSet<DataElementOperand>();
+ allOperands.addAll( nonCalculatedOperands );
+ allOperands.addAll( indicatorOperands );
+ allOperands.addAll( calculatedOperands );
final Collection<DataElementOperand> sumIntDataElementOperands = ParserUtil.filterOperands(
- nonCalculatedDataElementOperands, VALUE_TYPE_INT, AGGREGATION_OPERATOR_SUM );
+ nonCalculatedOperands, VALUE_TYPE_INT, AGGREGATION_OPERATOR_SUM );
final Collection<DataElementOperand> averageIntDataElementOperands = ParserUtil.filterOperands(
- nonCalculatedDataElementOperands, VALUE_TYPE_INT, AGGREGATION_OPERATOR_AVERAGE );
+ nonCalculatedOperands, VALUE_TYPE_INT, AGGREGATION_OPERATOR_AVERAGE );
final Collection<DataElementOperand> sumBoolDataElementOperands = ParserUtil.filterOperands(
- nonCalculatedDataElementOperands, VALUE_TYPE_BOOL, AGGREGATION_OPERATOR_SUM );
+ nonCalculatedOperands, VALUE_TYPE_BOOL, AGGREGATION_OPERATOR_SUM );
final Collection<DataElementOperand> averageBoolDataElementOperands = ParserUtil.filterOperands(
- nonCalculatedDataElementOperands, VALUE_TYPE_BOOL, AGGREGATION_OPERATOR_AVERAGE );
+ nonCalculatedOperands, VALUE_TYPE_BOOL, AGGREGATION_OPERATOR_AVERAGE );
log.info( "Filtered data elements: " + TimeUtils.getHMS() );
@@ -270,9 +280,9 @@
String key = RandomStringUtils.randomAlphanumeric( 8 );
- if ( crossTabService.validateCrossTabTable( allDataElementOperands ) != 0 )
+ if ( crossTabService.validateCrossTabTable( allOperands ) != 0 )
{
- int excess = crossTabService.validateCrossTabTable( allDataElementOperands );
+ int excess = crossTabService.validateCrossTabTable( allOperands );
log.warn( "Cannot crosstabulate since the number of data elements exceeded maximum columns: " + excess );
@@ -291,7 +301,7 @@
Collection<Integer> intersectingPeriodIds = ConversionUtils.getIdentifiers( Period.class, periodService.getIntersectionPeriods( periods ) );
final Collection<DataElementOperand> emptyOperands = crossTabService.populateCrossTabTable(
- allDataElementOperands, intersectingPeriodIds, childrenIds, key );
+ allOperands, intersectingPeriodIds, childrenIds, key );
log.info( "Populated crosstab table: " + TimeUtils.getHMS() );
@@ -363,8 +373,7 @@
if ( indicators != null && indicators.size() > 0 )
{
- count += indicatorDataMart.exportIndicatorValues( indicators, periods, organisationUnits,
- dataElementInIndicatorOperands, key );
+ count += indicatorDataMart.exportIndicatorValues( indicators, periods, organisationUnits, indicatorOperands, key );
log.info( "Exported values for indicators (" + indicators.size() + "): " + TimeUtils.getHMS() );
}
@@ -377,11 +386,9 @@
if ( calculatedDataElements != null && calculatedDataElements.size() > 0 )
{
- count += calculatedDataElementDataMart.exportCalculatedDataElements( calculatedDataElements, periods,
- organisationUnits, dataElementInCalculatedDataElementOperands, key );
+ count += calculatedDataElementDataMart.exportCalculatedDataElements( calculatedDataElements, periods, organisationUnits, calculatedOperands, key );
- log.info( "Exported values for calculated data elements (" + calculatedDataElements.size() + "): "
- + TimeUtils.getHMS() );
+ log.info( "Exported values for calculated data elements (" + calculatedDataElements.size() + "): " + TimeUtils.getHMS() );
}
crossTabService.dropCrossTabTable( key );
@@ -399,41 +406,35 @@
// Supportive methods
// -------------------------------------------------------------------------
- /**
- * Returns all data element identifiers included in the indicators in the
- * given identifier collection.
- */
- private Set<Integer> getDataElementIdsInIndicators( final Collection<Indicator> indicators )
+ private Set<DataElementOperand> getOperandsInIndicators( Collection<Indicator> indicators )
{
- final Set<Integer> identifiers = new HashSet<Integer>( indicators.size() );
-
- for ( final Indicator indicator : indicators )
+ final Set<DataElementOperand> operands = new HashSet<DataElementOperand>();
+
+ for ( Indicator indicator : indicators )
{
- identifiers.addAll( getDataElementIdsInExpression( indicator.getNumerator() ) );
- identifiers.addAll( getDataElementIdsInExpression( indicator.getDenominator() ) );
+ Set<DataElementOperand> temp = expressionService.getOperandsInExpression( indicator.getNumerator() );
+ operands.addAll( temp != null ? temp : new HashSet<DataElementOperand>() );
+
+ temp = expressionService.getOperandsInExpression( indicator.getDenominator() );
+ operands.addAll( temp != null ? temp : new HashSet<DataElementOperand>() );
}
-
- return identifiers;
+
+ return operands;
}
-
- /**
- * Returns all data element identifiers included in the calculated data
- * elements in the given identifier collection.
- */
- private Set<Integer> getDataElementIdsInCalculatedDataElements( final Collection<CalculatedDataElement> calculatedDataElements )
+
+ private Set<DataElementOperand> getOperandsInCalculatedDataElements( final Collection<CalculatedDataElement> calculatedDataElements )
{
- final Set<Integer> identifiers = new HashSet<Integer>();
-
+ final Set<DataElementOperand> operands = new HashSet<DataElementOperand>();
+
for ( final CalculatedDataElement calculatedDataElement : calculatedDataElements )
{
- final Set<DataElement> dataElements = expressionService.getDataElementsInCalculatedDataElement( calculatedDataElement );
-
- if ( dataElements != null )
+ if ( calculatedDataElement != null && calculatedDataElement.getExpression() != null )
{
- identifiers.addAll( ConversionUtils.getIdentifiers( DataElement.class, dataElements ) );
+ Set<DataElementOperand> temp = expressionService.getOperandsInExpression( calculatedDataElement.getExpression().getExpression() );
+ operands.addAll( temp != null ? temp : new HashSet<DataElementOperand>() );
}
}
-
- return identifiers;
+
+ return operands;
}
}
