launchpad-reviewers team mailing list archive

[Tim Penhey <tim.penhey@xxxxxxxxxxxxx>]

Tim Penhey has proposed merging lp:~thumper/launchpad/code-import-event-garbo into lp:launchpad/devel with lp:~thumper/launchpad/move-garbo-to-lp as a prerequisite.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)
Related bugs:
  #631301 Historical code import result tables need pruning
  https://bugs.launchpad.net/bugs/631301


Add a garbo pruner for the CodeImportEvent table.
-- 
https://code.launchpad.net/~thumper/launchpad/code-import-event-garbo/+merge/38675
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~thumper/launchpad/code-import-event-garbo into lp:launchpad/devel.

=== modified file 'database/schema/security.cfg'
--- database/schema/security.cfg	2010-10-08 10:06:11 +0000
+++ database/schema/security.cfg	2010-10-17 22:58:46 +0000
@@ -1926,6 +1926,8 @@
 public.bugtag                           = SELECT
 public.bugwatch                         = SELECT, UPDATE
 public.bugwatchactivity                 = SELECT, DELETE
+public.codeimportevent                  = SELECT, DELETE
+public.codeimporteventdata              = SELECT, DELETE
 public.codeimportresult                 = SELECT, DELETE
 public.emailaddress                     = SELECT
 public.oauthnonce                       = SELECT, DELETE

=== modified file 'lib/lp/code/model/codeimportevent.py'
--- lib/lp/code/model/codeimportevent.py	2010-08-20 20:31:18 +0000
+++ lib/lp/code/model/codeimportevent.py	2010-10-17 22:58:46 +0000
@@ -20,7 +20,7 @@
     )
 from zope.interface import implements
 
-from canonical.database.constants import DEFAULT
+from canonical.database.constants import DEFAULT, UTC_NOW
 from canonical.database.datetimecol import UtcDateTimeCol
 from canonical.database.enumcol import EnumCol
 from canonical.database.sqlbase import SQLBase
@@ -144,12 +144,14 @@
                 data_type=CodeImportEventDataType.MESSAGE,
                 data_value=message)
 
-    def newOnline(self, machine, user=None, message=None):
+    def newOnline(self, machine, user=None, message=None, _date_created=None):
         """See `ICodeImportEventSet`."""
         assert machine is not None, "machine must not be None"
+        if _date_created is None:
+            _date_created = UTC_NOW
         event = CodeImportEvent(
             event_type=CodeImportEventType.ONLINE,
-            machine=machine, person=user)
+            machine=machine, person=user, date_created=_date_created)
         self._recordMessage(event, message)
         return event
 

=== modified file 'lib/lp/scripts/garbo.py'
--- lib/lp/scripts/garbo.py	2010-10-03 15:30:06 +0000
+++ lib/lp/scripts/garbo.py	2010-10-17 22:58:46 +0000
@@ -4,7 +4,10 @@
 """Database garbage collection."""
 
 __metaclass__ = type
-__all__ = ['DailyDatabaseGarbageCollector', 'HourlyDatabaseGarbageCollector']
+__all__ = [
+    'DailyDatabaseGarbageCollector',
+    'HourlyDatabaseGarbageCollector',
+    ]
 
 from datetime import (
     datetime,
@@ -18,6 +21,7 @@
     In,
     Max,
     Min,
+    Select,
     SQL,
     )
 import transaction
@@ -58,6 +62,7 @@
     )
 from lp.code.interfaces.revision import IRevisionSet
 from lp.code.model.branchjob import BranchJob
+from lp.code.model.codeimportevent import CodeImportEvent
 from lp.code.model.codeimportresult import CodeImportResult
 from lp.code.model.revision import (
     RevisionAuthor,
@@ -195,6 +200,35 @@
         transaction.commit()
 
 
+class CodeImportEventPruner(TunableLoop):
+    """Prune `CodeImportEvent`s that are more than a month old.
+
+    Events that happened more than 30 days ago are really of no interest to us.
+    """
+
+    maximum_chunk_size = 10000
+    minimum_chunk_size = 500
+
+    def isDone(self):
+        store = IMasterStore(CodeImportEvent)
+        events = store.find(
+            CodeImportEvent,
+            CodeImportEvent.date_created < THIRTY_DAYS_AGO)
+        return events.any() is None
+
+    def __call__(self, chunk_size):
+        chunk_size = int(chunk_size)
+        store = IMasterStore(CodeImportEvent)
+        event_ids = Select(
+            [CodeImportEvent.id],
+            CodeImportEvent.date_created < THIRTY_DAYS_AGO,
+            limit=chunk_size)
+        num_removed = store.find(
+            CodeImportEvent, CodeImportEvent.id.is_in(event_ids)).remove()
+        transaction.commit()
+        self.log.debug("Removed %d old CodeImportEvents" % num_removed)
+
+
 class CodeImportResultPruner(TunableLoop):
     """A TunableLoop to prune unwanted CodeImportResult rows.
 
@@ -850,6 +884,7 @@
         BranchJobPruner,
         BugNotificationPruner,
         BugWatchActivityPruner,
+        CodeImportEventPruner,
         CodeImportResultPruner,
         HWSubmissionEmailLinker,
         ObsoleteBugAttachmentDeleter,

=== modified file 'lib/lp/scripts/tests/test_garbo.py'
--- lib/lp/scripts/tests/test_garbo.py	2010-10-17 22:58:44 +0000
+++ lib/lp/scripts/tests/test_garbo.py	2010-10-17 22:58:46 +0000
@@ -60,10 +60,12 @@
     RepositoryFormat,
     )
 from lp.code.enums import CodeImportResultStatus
+from lp.code.interfaces.codeimportevent import ICodeImportEventSet
 from lp.code.model.branchjob import (
     BranchJob,
     BranchUpgradeJob,
     )
+from lp.code.model.codeimportevent import CodeImportEvent
 from lp.code.model.codeimportresult import CodeImportResult
 from lp.registry.interfaces.person import (
     IPersonSet,
@@ -257,6 +259,34 @@
                 Min(CodeImportResult.date_created)).one().replace(tzinfo=UTC)
             >= now - timedelta(days=30))
 
+    def test_CodeImportEventPruner(self):
+        now = datetime.utcnow().replace(tzinfo=UTC)
+        store = IMasterStore(CodeImportResult)
+
+        LaunchpadZopelessLayer.switchDbUser('testadmin')
+        machine = self.factory.makeCodeImportMachine()
+        requester = self.factory.makePerson()
+        # Create 6 code import events for this machine, 3 on each side of 30
+        # days. Use the event set to the extra event data rows get created too.
+        event_set = getUtility(ICodeImportEventSet)
+        for age in (35, 33, 31, 29, 27, 15):
+            event_set.newOnline(
+                machine, user=requester, message='Hello',
+                _date_created=(now - timedelta(days=age)))
+        transaction.commit()
+
+        # Run the garbage collector
+        self.runDaily()
+
+        # Only the three most recent results are left.
+        events = list(machine.events)
+        self.assertEqual(3, len(events))
+        # We now have no CodeImportEvents older than 30 days
+        self.failUnless(
+            store.find(
+                Min(CodeImportEvent.date_created)).one().replace(tzinfo=UTC)
+            >= now - timedelta(days=30))
+
     def test_OpenIDConsumerAssociationPruner(self):
         pruner = OpenIDConsumerAssociationPruner
         table_name = pruner.table_name