dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #31635
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 16150: Impl support for data element totals in validation rules
------------------------------------------------------------
revno: 16150
committer: Lars Helge Overland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Wed 2014-07-16 17:29:27 +0200
message:
Impl support for data element totals in validation rules
modified:
dhis-2/dhis-api/src/main/java/org/hisp/dhis/expression/Expression.java
dhis-2/dhis-api/src/main/java/org/hisp/dhis/expression/ExpressionService.java
dhis-2/dhis-api/src/main/java/org/hisp/dhis/indicator/Indicator.java
dhis-2/dhis-api/src/main/java/org/hisp/dhis/validation/ValidationResult.java
dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/DefaultExpressionService.java
dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidatorThread.java
dhis-2/dhis-web/dhis-web-validationrule/src/main/webapp/dhis-web-validationrule/javascript/expressionBuilder.js
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/expression/Expression.java'
--- dhis-2/dhis-api/src/main/java/org/hisp/dhis/expression/Expression.java 2014-03-18 08:10:10 +0000
+++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/expression/Expression.java 2014-07-16 15:29:27 +0000
@@ -28,12 +28,14 @@
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
+import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonView;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
+
import org.apache.commons.lang.Validate;
import org.hisp.dhis.common.BaseIdentifiableObject;
import org.hisp.dhis.common.DxfNamespaces;
@@ -113,6 +115,12 @@
private Set<DataElementCategoryOptionCombo> optionCombosInExpression = new HashSet<DataElementCategoryOptionCombo>();
// -------------------------------------------------------------------------
+ // Transient properties
+ // -------------------------------------------------------------------------
+
+ private transient String explodedExpression;
+
+ // -------------------------------------------------------------------------
// Constructors
// -------------------------------------------------------------------------
@@ -140,6 +148,18 @@
}
// -------------------------------------------------------------------------
+ // Logic
+ // -------------------------------------------------------------------------
+
+ /**
+ * Returns exploded expression, if null returns expression.
+ */
+ public String getExplodedExpressionFallback()
+ {
+ return explodedExpression != null ? explodedExpression : expression;
+ }
+
+ // -------------------------------------------------------------------------
// Equals and hashCode
// -------------------------------------------------------------------------
@@ -195,8 +215,8 @@
{
final int PRIME = 31;
int result = 1;
- result = PRIME * result + ((description == null) ? 0 : description.hashCode());
- result = PRIME * result + ((expression == null) ? 0 : expression.hashCode());
+ result = PRIME * result + ( ( description == null ) ? 0 : description.hashCode() );
+ result = PRIME * result + ( ( expression == null ) ? 0 : expression.hashCode() );
return result;
}
@@ -207,6 +227,7 @@
return "Expression{" +
"id=" + id +
", expression='" + expression + '\'' +
+ ", explodedExpression='" + explodedExpression + '\'' +
", description='" + description + '\'' +
", dataElementsInExpression=" + dataElementsInExpression.size() +
", optionCombosInExpression=" + optionCombosInExpression.size() +
@@ -296,6 +317,17 @@
this.nullIfBlank = nullIfBlank;
}
+ @JsonIgnore
+ public String getExplodedExpression()
+ {
+ return explodedExpression;
+ }
+
+ public void setExplodedExpression( String explodedExpression )
+ {
+ this.explodedExpression = explodedExpression;
+ }
+
public void mergeWith( Expression other )
{
Validate.notNull( other );
=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/expression/ExpressionService.java'
--- dhis-2/dhis-api/src/main/java/org/hisp/dhis/expression/ExpressionService.java 2014-03-18 08:10:10 +0000
+++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/expression/ExpressionService.java 2014-07-16 15:29:27 +0000
@@ -39,6 +39,7 @@
import org.hisp.dhis.indicator.Indicator;
import org.hisp.dhis.organisationunit.OrganisationUnitGroup;
import org.hisp.dhis.period.Period;
+import org.hisp.dhis.validation.ValidationRule;
/**
* Expressions are mathematical formulas and can contain references to various
@@ -277,6 +278,9 @@
* Populates the explodedNumerator and explodedDenominator property on all
* indicators in the given collection. This method uses
* explodeExpression( String ) internally to generate the exploded expressions.
+ * Replaces references to data element totals with references to all
+ * category option combos in the category combo for that data element.
+ *
* This method will perform better compared to calling explodeExpression( String )
* multiple times outside a transactional context as the transactional
* overhead is avoided.
@@ -290,12 +294,25 @@
* Populates the explodedNumerator and explodedDenominator property on all
* indicators in the given collection. This method uses
* explodeExpression( String ) internally to generate the exploded expressions.
+ * Replaces references to data element totals with references to all
+ * category option combos in the category combo for that data element.
*
* @param indicators the collection of indicators.
*/
void explodeExpressions( Collection<Indicator> indicators );
/**
+ * Populates the explodedExpression property on the Expression object of all
+ * validation rules in the given collection. This method uses
+ * explodeExpression( String ) internally to generate the exploded expressions.
+ * Replaces references to data element totals with references to all
+ * category option combos in the category combo for that data element.
+ *
+ * @param validationRules the collection of validation rules.
+ */
+ void explodeValidationRuleExpressions( Collection<ValidationRule> validationRules );
+
+ /**
* Replaces references to data element totals with references to all
* category option combos in the category combo for that data element.
*
=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/indicator/Indicator.java'
--- dhis-2/dhis-api/src/main/java/org/hisp/dhis/indicator/Indicator.java 2014-04-25 11:22:12 +0000
+++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/indicator/Indicator.java 2014-07-16 15:29:27 +0000
@@ -41,6 +41,7 @@
import org.hisp.dhis.dataset.DataSet;
import org.hisp.dhis.mapping.MapLegendSet;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonView;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
@@ -68,13 +69,13 @@
private String numeratorDescription;
- private String explodedNumerator;
+ private transient String explodedNumerator;
private String denominator;
private String denominatorDescription;
- private String explodedDenominator;
+ private transient String explodedDenominator;
private Integer sortOrder;
@@ -222,9 +223,7 @@
this.numeratorDescription = numeratorDescription;
}
- @JsonProperty
- @JsonView( {DetailedView.class, ExportView.class} )
- @JacksonXmlProperty( namespace = DxfNamespaces.DXF_2_0)
+ @JsonIgnore
public String getExplodedNumerator()
{
return explodedNumerator;
@@ -261,9 +260,7 @@
this.denominatorDescription = denominatorDescription;
}
- @JsonProperty
- @JsonView( {DetailedView.class, ExportView.class} )
- @JacksonXmlProperty( namespace = DxfNamespaces.DXF_2_0)
+ @JsonIgnore
public String getExplodedDenominator()
{
return explodedDenominator;
=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/validation/ValidationResult.java'
--- dhis-2/dhis-api/src/main/java/org/hisp/dhis/validation/ValidationResult.java 2014-05-18 00:49:40 +0000
+++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/validation/ValidationResult.java 2014-07-16 15:29:27 +0000
@@ -95,11 +95,11 @@
return result;
}
- //
- // Note: this method is called from threads in which it may not be possible
- // to initialize lazy Hibernate properties. So object properties to compare
- // must be chosen accordingly.
- //
+ /**
+ * Note: this method is called from threads in which it may not be possible
+ * to initialize lazy Hibernate properties. So object properties to compare
+ * must be chosen accordingly.
+ */
@Override
public boolean equals( Object object )
{
@@ -203,11 +203,11 @@
return true;
}
- //
- // Note: this method is called from threads in which it may not be possible
- // to initialize lazy Hibernate properties. So object properties to compare
- // must be chosen accordingly.
- //
+ /**
+ * Note: this method is called from threads in which it may not be possible
+ * to initialize lazy Hibernate properties. So object properties to compare
+ * must be chosen accordingly.
+ */
public int compareTo( ValidationResult other )
{
int result = source.getName().compareTo( other.source.getName() );
@@ -291,7 +291,11 @@
@Override
public String toString()
{
- return source + " - " + period + " - " + attributeOptionCombo.getName() + " - " + validationRule + " - " + leftsideValue + " - " + rightsideValue;
+ return "[Source: " + source +
+ ", period: " + period +
+ ", validation rule: " + validationRule +
+ ", left side value: " + leftsideValue +
+ ", right side value: " + rightsideValue + "]";
}
// -------------------------------------------------------------------------
=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/DefaultExpressionService.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/DefaultExpressionService.java 2014-03-18 08:10:10 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/DefaultExpressionService.java 2014-07-16 15:29:27 +0000
@@ -63,6 +63,7 @@
import org.hisp.dhis.period.Period;
import org.hisp.dhis.system.util.DateUtils;
import org.hisp.dhis.system.util.MathUtils;
+import org.hisp.dhis.validation.ValidationRule;
import org.springframework.transaction.annotation.Transactional;
/**
@@ -196,7 +197,7 @@
public Double getExpressionValue( Expression expression, Map<DataElementOperand, Double> valueMap,
Map<String, Double> constantMap, Map<String, Integer> orgUnitCountMap, Integer days )
{
- final String expressionString = generateExpression( expression.getExpression(), valueMap, constantMap,
+ final String expressionString = generateExpression( expression.getExplodedExpressionFallback(), valueMap, constantMap,
orgUnitCountMap, days, expression.isNullIfBlank() );
return expressionString != null ? calculateExpression( expressionString ) : null;
@@ -205,7 +206,7 @@
public Double getExpressionValue( Expression expression, Map<DataElementOperand, Double> valueMap,
Map<String, Double> constantMap, Map<String, Integer> orgUnitCountMap, Integer days, Set<DataElementOperand> incompleteValues )
{
- final String expressionString = generateExpression( expression.getExpression(), valueMap, constantMap, orgUnitCountMap, days,
+ final String expressionString = generateExpression( expression.getExplodedExpressionFallback(), valueMap, constantMap, orgUnitCountMap, days,
expression.isNullIfBlank(), incompleteValues );
return expressionString != null ? calculateExpression( expressionString ) : null;
@@ -603,7 +604,7 @@
explodeExpressions( indicators );
}
}
-
+
@Transactional
public void explodeExpressions( Collection<Indicator> indicators )
{
@@ -632,7 +633,36 @@
}
}
}
-
+
+ @Transactional
+ public void explodeValidationRuleExpressions( Collection<ValidationRule> validationRules )
+ {
+ if ( validationRules != null && !validationRules.isEmpty() )
+ {
+ Set<String> dataElementTotals = new HashSet<String>();
+
+ for ( ValidationRule rule : validationRules )
+ {
+ dataElementTotals.addAll( getDataElementTotalUids( rule.getLeftSide().getExpression() ) );
+ dataElementTotals.addAll( getDataElementTotalUids( rule.getRightSide().getExpression() ) );
+ }
+
+ if ( !dataElementTotals.isEmpty() )
+ {
+ final ListMap<String, String> dataElementMap = dataElementService.getDataElementCategoryOptionComboMap( dataElementTotals );
+
+ if ( !dataElementMap.isEmpty() )
+ {
+ for ( ValidationRule rule : validationRules )
+ {
+ rule.getLeftSide().setExplodedExpression( explodeExpression( rule.getLeftSide().getExplodedExpressionFallback(), dataElementMap ) );
+ rule.getRightSide().setExplodedExpression( explodeExpression( rule.getRightSide().getExplodedExpressionFallback(), dataElementMap ) );
+ }
+ }
+ }
+ }
+ }
+
private String explodeExpression( String expression, ListMap<String, String> dataElementOptionComboMap )
{
if ( expression == null || expression.isEmpty() )
@@ -655,7 +685,7 @@
for ( String coc : cocs )
{
- replace.append( EXP_OPEN ).append( matcher.group( 1 ) ).append( SEPARATOR ).append(
+ replace.append( EXP_OPEN ).append( de ).append( SEPARATOR ).append(
coc ).append( EXP_CLOSE ).append( "+" );
}
=== modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidatorThread.java'
--- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidatorThread.java 2014-07-16 12:15:52 +0000
+++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidatorThread.java 2014-07-16 15:29:27 +0000
@@ -92,7 +92,8 @@
for ( PeriodTypeExtended periodTypeX : context.getPeriodTypeExtendedMap().values() )
{
Collection<DataElement> sourceDataElements = periodTypeX.getSourceDataElements().get( sourceX.getSource() );
- Set<ValidationRule> rules = getRulesBySourceAndPeriodType( sourceX, periodTypeX, sourceDataElements );
+ Set<ValidationRule> rules = getRulesBySourceAndPeriodType( sourceX, periodTypeX, sourceDataElements );
+ context.getExpressionService().explodeValidationRuleExpressions( rules );
if ( !rules.isEmpty() )
{
@@ -112,7 +113,7 @@
for ( ValidationRule rule : rules )
{
- if ( evaluateCheck( currentValueMap, lastUpdatedMap, rule ) )
+ if ( evaluateValidationCheck( currentValueMap, lastUpdatedMap, rule ) )
{
Map<Integer, Double> leftSideValues = getExpressionValueMap( rule.getLeftSide(),
currentValueMap, incompleteValuesMap );
@@ -120,19 +121,19 @@
if ( !leftSideValues.isEmpty() || Operator.compulsory_pair.equals( rule.getOperator() ) )
{
Map<Integer, Double> rightSideValues = getRightSideValue( sourceX.getSource(), periodTypeX, period, rule,
- currentValueMap, sourceDataElements );
+ currentValueMap, sourceDataElements );
if ( !rightSideValues.isEmpty() || Operator.compulsory_pair.equals( rule.getOperator() ) )
{
- Set<Integer> combos = leftSideValues.keySet();
+ Set<Integer> attributeOptionCombos = leftSideValues.keySet();
if ( Operator.compulsory_pair.equals( rule.getOperator() ) )
{
- combos = new HashSet<Integer>( combos );
- combos.addAll( rightSideValues.keySet() );
+ attributeOptionCombos = new HashSet<Integer>( attributeOptionCombos );
+ attributeOptionCombos.addAll( rightSideValues.keySet() );
}
- for ( int combo : combos )
+ for ( int combo : attributeOptionCombos )
{
Double leftSide = leftSideValues.get ( combo );
Double rightSide = rightSideValues.get ( combo );
@@ -197,6 +198,7 @@
// But if this is some funny kind of rule with no elements
// (like for testing), include it also.
Collection<DataElement> elements = rule.getCurrentDataElements();
+
if ( elements == null || elements.size() == 0 || sourceDataElements.containsAll( elements ) )
{
periodTypeRules.add( rule );
@@ -233,7 +235,7 @@
* @param rule the rule that may be evaluated
* @return true if the rule should be evaluated with this data, false if not
*/
- private boolean evaluateCheck( MapMap<Integer, DataElementOperand, Double> currentValueMapMap,
+ private boolean evaluateValidationCheck( MapMap<Integer, DataElementOperand, Double> currentValueMapMap,
MapMap<Integer, DataElementOperand, Date> lastUpdatedMapMap, ValidationRule rule )
{
boolean evaluate = true; // Assume true for now.
@@ -268,6 +270,7 @@
for ( DataElementOperand deo : deos )
{
Date lastUpdated = entry.getValue().get( deo );
+
if ( lastUpdated != null && lastUpdated.after( context.getLastScheduledRun() ) )
{
saveThisCombo = true; // True if new/updated data.
@@ -454,17 +457,17 @@
* @return map of values.
*/
private Map<Integer, Double> getExpressionValueMap( Expression expression,
- MapMap<Integer, DataElementOperand, Double> valueMap,
- SetMap<Integer, DataElementOperand> incompleteValuesMap )
+ MapMap<Integer, DataElementOperand, Double> valueMap,
+ SetMap<Integer, DataElementOperand> incompleteValuesMap )
{
Map<Integer, Double> expressionValueMap = new HashMap<Integer, Double>();
- for ( Map.Entry<Integer, Map<DataElementOperand, Double>> e : valueMap.entrySet() )
+ for ( Map.Entry<Integer, Map<DataElementOperand, Double>> entry : valueMap.entrySet() )
{
- expressionValueMap.put( e.getKey(),
+ expressionValueMap.put( entry.getKey(),
context.getExpressionService().getExpressionValue( expression,
- e.getValue(), context.getConstantMap(), null, null,
- incompleteValuesMap.getSet( e.getKey() ) ) );
+ entry.getValue(), context.getConstantMap(), null, null,
+ incompleteValuesMap.getSet( entry.getKey() ) ) );
}
return expressionValueMap;
@@ -480,8 +483,8 @@
* @param sequentialSampleCount number of sequential samples tried for
* @return average right-side sample value
*/
- private Double rightSideAverage( ValidationRule rule, List<Double> sampleValues, int annualSampleCount,
- int sequentialSampleCount )
+ private Double rightSideAverage( ValidationRule rule, List<Double> sampleValues,
+ int annualSampleCount, int sequentialSampleCount )
{
// Find the expected sample count for the last period of its type in the
// database: sequentialSampleCount for the immediately preceding periods
@@ -546,10 +549,10 @@
* @return map of attribute option combo to map of values found.
*/
private MapMap<Integer, DataElementOperand, Double> getValueMap( PeriodTypeExtended periodTypeX,
- Collection<DataElement> ruleDataElements, Collection<DataElement> sourceDataElements,
- Set<DataElement> recursiveDataElements, Collection<PeriodType> allowedPeriodTypes, Period period,
- OrganisationUnit source, MapMap<Integer, DataElementOperand, Date> lastUpdatedMap,
- SetMap<Integer, DataElementOperand> incompleteValuesMap )
+ Collection<DataElement> ruleDataElements, Collection<DataElement> sourceDataElements,
+ Set<DataElement> recursiveDataElements, Collection<PeriodType> allowedPeriodTypes, Period period,
+ OrganisationUnit source, MapMap<Integer, DataElementOperand, Date> lastUpdatedMap,
+ SetMap<Integer, DataElementOperand> incompleteValuesMap )
{
Set<DataElement> dataElementsToGet = new HashSet<DataElement>( ruleDataElements );
dataElementsToGet.retainAll( sourceDataElements );
=== modified file 'dhis-2/dhis-web/dhis-web-validationrule/src/main/webapp/dhis-web-validationrule/javascript/expressionBuilder.js'
--- dhis-2/dhis-web/dhis-web-validationrule/src/main/webapp/dhis-web-validationrule/javascript/expressionBuilder.js 2013-10-16 12:39:47 +0000
+++ dhis-2/dhis-web/dhis-web-validationrule/src/main/webapp/dhis-web-validationrule/javascript/expressionBuilder.js 2014-07-16 15:29:27 +0000
@@ -73,7 +73,7 @@
var periodTypeAllowAverage = ( ruleType && ruleType == "surveillance" ) ? true : false;
dataDictionary.loadOperands( "#expression-container select[id=dataElementId]",
- {usePaging: true, key: key, periodType: periodType, periodTypeAllowAverage: periodTypeAllowAverage } );
+ {usePaging: true, key: key, periodType: periodType, includeTotals: true, periodTypeAllowAverage: periodTypeAllowAverage } );
}
function clearSearchText()
