dhis2-devs team mailing list archive

[noreply@xxxxxxxxxxxxx]

------------------------------------------------------------
revno: 9391
committer: Lars Helge Øverland <larshelge@xxxxxxxxx>
branch nick: dhis2
timestamp: Sat 2012-12-22 19:09:46 +0100
message:
  Impl vacuum after table update
modified:
  dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java
  dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java
  dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java
  dhis-2/dhis-support/dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc/statementbuilder/MySQLStatementBuilder.java


--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk

Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription

=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java	2012-12-22 17:51:57 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java	2012-12-22 18:09:46 +0000
@@ -119,4 +119,12 @@
      * @param aggregationLevel the aggregation level.
      */
     void applyAggregationLevels( String tableName, Collection<String> dataElements, int aggregationLevel );
+    
+    /**
+     * Performs vacuum or optimization of the given table. The type of operation
+     * performed is dependent on the underlying DBMS.
+     * 
+     * @param tableName the name of the analytics table.
+     */
+    Future<?> vacuumTableAsync( String tableName );
 }

=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java	2012-12-22 17:51:57 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java	2012-12-22 18:09:46 +0000
@@ -98,6 +98,9 @@
         createIndexes( tables );
         clock.logTime( "Created all indexes" );
         
+        vacuumTables( tables );
+        clock.logTime( "Vacuumed tables" );
+        
         swapTables( tables );
         clock.logTime( "Swapped analytics tables" );
         
@@ -120,11 +123,7 @@
     
     private void populateTables( List<String> tables )
     {
-        int pageSize = Math.max( ( SystemUtils.getCpuCores() - 1 ), 1 );
-
-        log.info( "Page size: " + pageSize );
-        
-        List<List<String>> tablePages = new PaginatedList<String>( tables ).setPageSize( pageSize ).getPages();
+        List<List<String>> tablePages = new PaginatedList<String>( tables ).setPageSize( getProcessNo() ).getPages();
         
         log.info( "No of pages: " + tablePages.size() );
         
@@ -177,17 +176,30 @@
         }
     }
     
+    private void vacuumTables( List<String> tables )
+    {
+        List<List<String>> tablePages = new PaginatedList<String>( tables ).setPageSize( getProcessNo() ).getPages();
+        
+        for ( List<String> tablePage : tablePages )
+        {
+            List<Future<?>> futures = new ArrayList<Future<?>>();
+            
+            for ( String table : tablePage )
+            {
+                futures.add( tableManager.vacuumTableAsync( table ) );
+            }
+            
+            ConcurrentUtils.waitForCompletion( futures );
+        }
+    }
+    
     private void createIndexes( List<String> tables )
     {
-        int pages = Math.max( ( SystemUtils.getCpuCores() - 1 ), 1 );
-        
-        log.info( "No of pages: " + pages );
-        
         for ( String table : tables )
         {
             List<Future<?>> futures = new ArrayList<Future<?>>();
     
-            List<List<String>> columnPages = new PaginatedList<String>( tableManager.getDimensionColumnNames() ).setNumberOfPages( pages ).getPages();
+            List<List<String>> columnPages = new PaginatedList<String>( tableManager.getDimensionColumnNames() ).setNumberOfPages( getProcessNo() ).getPages();
             
             for ( List<String> columnPage : columnPages )
             {
@@ -213,4 +225,9 @@
             tableManager.dropTable( table );
         }
     }
+    
+    private int getProcessNo()
+    {
+        return Math.max( ( SystemUtils.getCpuCores() - 1 ), 1 );
+    }
 }

=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java'
--- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java	2012-12-22 17:51:57 +0000
+++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java	2012-12-22 18:09:46 +0000
@@ -41,6 +41,7 @@
 import org.hisp.dhis.common.CodeGenerator;
 import org.hisp.dhis.dataelement.DataElementGroupSet;
 import org.hisp.dhis.dataelement.DataElementService;
+import org.hisp.dhis.jdbc.StatementBuilder;
 import org.hisp.dhis.organisationunit.OrganisationUnitGroupService;
 import org.hisp.dhis.organisationunit.OrganisationUnitGroupSet;
 import org.hisp.dhis.organisationunit.OrganisationUnitLevel;
@@ -89,6 +90,9 @@
     private OrganisationUnitGroupService organisationUnitGroupService;
    
     @Autowired
+    private StatementBuilder statementBuilder;
+    
+    @Autowired
     private JdbcTemplate jdbcTemplate;
     
     // -------------------------------------------------------------------------
@@ -340,7 +344,19 @@
         log.info( "Aggregation level SQL: " + sql.toString() );
         
         jdbcTemplate.execute( sql.toString() );
-    }   
+    }
+    
+    @Async
+    public Future<?> vacuumTableAsync( String tableName )
+    {
+        final String sql = statementBuilder.getVacuum( tableName );
+        
+        log.info( "Vacuum SQL:" + sql );
+        
+        jdbcTemplate.execute( sql );
+        
+        return null;
+    }
     
     // -------------------------------------------------------------------------
     // Supportive methods

=== modified file 'dhis-2/dhis-support/dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc/statementbuilder/MySQLStatementBuilder.java'
--- dhis-2/dhis-support/dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc/statementbuilder/MySQLStatementBuilder.java	2012-10-29 11:18:39 +0000
+++ dhis-2/dhis-support/dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc/statementbuilder/MySQLStatementBuilder.java	2012-12-22 18:09:46 +0000
@@ -44,7 +44,7 @@
     @Override
     public String getVacuum( String table )
     {
-        return null;
+        return "optimize table " + table + ";";
     }
 
     @Override