dhis2-devs team mailing list archive
-
dhis2-devs team
-
Mailing list archive
-
Message #15589
[Branch ~dhis2-devs-core/dhis2/trunk] Rev 5682: Data mart, improved logging
------------------------------------------------------------
revno: 5682
committer: Lars Helge Overland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Mon 2012-01-09 21:03:56 +0100
message:
Data mart, improved logging
modified:
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageBoolAggregator.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntSingleValueAggregator.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumBoolAggregator.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumIntAggregator.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DefaultDataElementDataMart.java
dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java
dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/SystemUtils.java
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageBoolAggregator.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageBoolAggregator.java 2011-12-26 10:07:59 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageBoolAggregator.java 2012-01-09 20:03:56 +0000
@@ -88,7 +88,7 @@
final Map<DataElementOperand, double[]> entries = getAggregate( crossTabValues, period.getStartDate(),
period.getEndDate(), period.getStartDate(), period.getEndDate(), unitLevel ); // <Operand, [total value, total relevant days]>
- final Map<DataElementOperand, Double> values = new HashMap<DataElementOperand, Double>( entries.size() ); // <Operand, total value>
+ final Map<DataElementOperand, Double> values = new HashMap<DataElementOperand, Double>(); // <Operand, total value>
double average = 0.0;
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntSingleValueAggregator.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntSingleValueAggregator.java 2011-12-26 10:07:59 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntSingleValueAggregator.java 2012-01-09 20:03:56 +0000
@@ -87,7 +87,7 @@
final Collection<CrossTabDataValue> crossTabValues = crossTabService.getCrossTabDataValues( operands,
aggregationCache.getIntersectingPeriods( period.getStartDate(), period.getEndDate() ), organisationUnits, key );
- final Map<DataElementOperand, Double> values = new HashMap<DataElementOperand, Double>( crossTabValues.size() * operands.size() ); // <Operand, total value>
+ final Map<DataElementOperand, Double> values = new HashMap<DataElementOperand, Double>(); // <Operand, total value>
for ( final CrossTabDataValue crossTabValue : crossTabValues )
{
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumBoolAggregator.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumBoolAggregator.java 2011-12-26 10:07:59 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumBoolAggregator.java 2012-01-09 20:03:56 +0000
@@ -83,7 +83,7 @@
final Collection<CrossTabDataValue> crossTabValues = crossTabService.getCrossTabDataValues( operands,
aggregationCache.getIntersectingPeriods( period.getStartDate(), period.getEndDate() ), organisationUnits, key );
- final Map<DataElementOperand, Double> values = new HashMap<DataElementOperand, Double>( crossTabValues.size() * operands.size() ); // <Operand, total value>
+ final Map<DataElementOperand, Double> values = new HashMap<DataElementOperand, Double>(); // <Operand, total value>
for ( final CrossTabDataValue crossTabValue : crossTabValues )
{
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumIntAggregator.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumIntAggregator.java 2011-12-26 10:07:59 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumIntAggregator.java 2012-01-09 20:03:56 +0000
@@ -87,7 +87,7 @@
final Collection<CrossTabDataValue> crossTabValues = crossTabService.getCrossTabDataValues( operands,
aggregationCache.getPeriodsBetweenDates( period.getStartDate(), period.getEndDate() ), organisationUnits, key );
- final Map<DataElementOperand, Double> values = new HashMap<DataElementOperand, Double>( crossTabValues.size() * operands.size() ); // <Operand, total value>
+ final Map<DataElementOperand, Double> values = new HashMap<DataElementOperand, Double>(); // <Operand, total value>
for ( final CrossTabDataValue crossTabValue : crossTabValues )
{
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java 2011-12-26 10:07:59 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java 2012-01-09 20:03:56 +0000
@@ -131,6 +131,8 @@
final BatchHandler<Object> batchHandler = batchHandlerFactory.createBatchHandler( GenericBatchHandler.class ).
setTableName( CrossTabStore.CROSSTAB_TABLE_PREFIX + key ).init();
+ int rows = 0;
+
for ( final Integer periodId : periodIds )
{
for ( final Integer sourceId : organisationUnitIds )
@@ -177,6 +179,8 @@
if ( hasValues )
{
+ rows++;
+
batchHandler.addObject( valueList );
}
}
@@ -186,6 +190,8 @@
statementManager.destroy();
+ log.info( "Crosstab table columns: " + ( operands.size() + 2 ) + ", rows: " + rows );
+
return null;
}
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DefaultDataElementDataMart.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DefaultDataElementDataMart.java 2011-12-26 10:07:59 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DefaultDataElementDataMart.java 2012-01-09 20:03:56 +0000
@@ -53,6 +53,7 @@
import org.hisp.dhis.organisationunit.OrganisationUnitGroup;
import org.hisp.dhis.organisationunit.OrganisationUnitHierarchy;
import org.hisp.dhis.period.Period;
+import org.hisp.dhis.system.util.SystemUtils;
import org.springframework.scheduling.annotation.Async;
/**
@@ -210,7 +211,7 @@
}
}
- log.debug( "Exported data values for period: " + period );
+ log.debug( "Exported data values for period: " + period + ", " + SystemUtils.getMemoryString() );
}
batchHandler.flush();
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2011-12-26 10:07:59 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2012-01-09 20:03:56 +0000
@@ -186,7 +186,7 @@
{
final int cpuCores = SystemUtils.getCpuCores();
- Clock clock = new Clock().startClock().logTime( "Data mart export process started, number of CPU cores: " + cpuCores );
+ Clock clock = new Clock().startClock().logTime( "Data mart export process started, number of CPU cores: " + cpuCores + ", " + SystemUtils.getMemoryString() );
// ---------------------------------------------------------------------
// Replace null with empty collection
@@ -262,7 +262,7 @@
indicatorOperands.retainAll( allOperands );
- clock.logTime( "Number of operands with data: " + allOperands.size() );
+ clock.logTime( "Number of operands with data: " + allOperands.size() + ", " + SystemUtils.getMemoryString() );
// ---------------------------------------------------------------------
// Create crosstabtable
@@ -286,7 +286,7 @@
ConcurrentUtils.waitForCompletion( crossTabFutures );
- clock.logTime( "Populated crosstab table" );
+ clock.logTime( "Populated crosstab table, " + SystemUtils.getMemoryString() );
final boolean isIndicators = indicators != null && indicators.size() > 0;
@@ -296,7 +296,7 @@
crossTabService.createAggregatedDataCache( indicatorOperands, key );
- clock.logTime( "Created aggregated data cache" );
+ clock.logTime( "Created aggregated data cache, number of indicator operands: " + indicatorOperands.size() + ", operands with data: " + allOperands.size() );
// ---------------------------------------------------------------------
// 2. Drop potential indexes
@@ -343,7 +343,7 @@
ConcurrentUtils.waitForCompletion( futures );
- clock.logTime( "Exported values for data element operands (" + allOperands.size() + "), number of pages: " + organisationUnitPages.size() );
+ clock.logTime( "Exported values for data element operands (" + allOperands.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString() );
}
// ---------------------------------------------------------------------
@@ -379,7 +379,7 @@
ConcurrentUtils.waitForCompletion( futures );
- clock.logTime( "Exported values for indicators (" + indicators.size() + "), number of pages: " + organisationUnitPages.size() );
+ clock.logTime( "Exported values for indicators (" + indicators.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString() );
}
// ---------------------------------------------------------------------
@@ -388,7 +388,7 @@
crossTabService.dropAggregatedDataCache( key );
- clock.logTime( "Dropped aggregated data cache" );
+ clock.logTime( "Dropped aggregated data cache, " + SystemUtils.getMemoryString() );
// ---------------------------------------------------------------------
// 8. Create potential indexes
@@ -466,7 +466,7 @@
ConcurrentUtils.waitForCompletion( futures );
- clock.logTime( "Exported values for data element operands (" + allOperands.size() + "), number of pages: " + organisationUnitPages.size() );
+ clock.logTime( "Exported values for data element operands (" + allOperands.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString() );
}
// ---------------------------------------------------------------------
@@ -502,7 +502,7 @@
ConcurrentUtils.waitForCompletion( futures );
- clock.logTime( "Exported values for indicators (" + indicators.size() + "), number of pages: " + organisationUnitPages.size() );
+ clock.logTime( "Exported values for indicators (" + indicators.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString() );
}
// ---------------------------------------------------------------------
@@ -511,7 +511,7 @@
crossTabService.dropAggregatedOrgUnitDataCache( key );
- clock.logTime( "Dropped aggregated org unit data cache" );
+ clock.logTime( "Dropped aggregated org unit data cache, " + SystemUtils.getMemoryString() );
// ---------------------------------------------------------------------
// 8. Create potential indexes
@@ -524,8 +524,7 @@
clock.logTime( "Created org unit indexes" );
}
- clock.logTime( "Aggregated organisation unit data export done" );
-
+ clock.logTime( "Aggregated organisation unit data export done" );
}
// ---------------------------------------------------------------------
=== modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/SystemUtils.java'
--- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/SystemUtils.java 2011-12-26 10:07:59 +0000
+++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/SystemUtils.java 2012-01-09 20:03:56 +0000
@@ -33,6 +33,8 @@
*/
public class SystemUtils
{
+ private static final int FACTOR_MB = 1024 * 1024;
+
public static boolean isRunningForUse()
{
try
@@ -51,4 +53,11 @@
{
return Runtime.getRuntime().availableProcessors();
}
+
+ public static String getMemoryString()
+ {
+ return "Mem used: " + ( Runtime.getRuntime().totalMemory() / FACTOR_MB ) +
+ ", max: " + ( Runtime.getRuntime().maxMemory() / FACTOR_MB ) +
+ ", free: " + ( Runtime.getRuntime().freeMemory() / FACTOR_MB );
+ }
}
