launchpad-reviewers team mailing list archive

[William Grant <me@xxxxxxxxxxxxxxxxxx>]

William Grant has proposed merging lp:~wgrant/launchpad/antibfjo-2-garbo into lp:launchpad with lp:~wgrant/launchpad/antibfjo-1-set as a prerequisite.

Commit message:
Add a garbo job to backfill existing BuildQueue rows.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~wgrant/launchpad/antibfjo-2-garbo/+merge/195887

IBuildFarmJobOld and its implementations (BuildPackageJob, SourcePackageRecipeBuildJob, TranslationTemplatesBuildJob) are finally being replaced with an FK from BuildQueue to BuildFarmJob. Additionally, the two fields of Job that are used by BuildQueue are being inlined, and the FK to Job dropped, for performance reasons.

This branch adds a garbo job to backfill existing BuildQueue rows.
-- 
https://code.launchpad.net/~wgrant/launchpad/antibfjo-2-garbo/+merge/195887
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~wgrant/launchpad/antibfjo-2-garbo into lp:launchpad.

=== modified file 'database/schema/security.cfg'
--- database/schema/security.cfg	2013-10-03 04:37:30 +0000
+++ database/schema/security.cfg	2013-11-20 00:21:46 +0000
@@ -2241,6 +2241,7 @@
 public.bugtaskflat                      = SELECT
 public.bugwatch                         = SELECT, UPDATE
 public.bugwatchactivity                 = SELECT, DELETE
+public.buildqueue                       = SELECT, UPDATE
 public.codeimportevent                  = SELECT, DELETE
 public.codeimporteventdata              = SELECT, DELETE
 public.codeimportresult                 = SELECT, DELETE

=== modified file 'lib/lp/scripts/garbo.py'
--- lib/lp/scripts/garbo.py	2013-06-20 05:50:00 +0000
+++ lib/lp/scripts/garbo.py	2013-11-20 00:21:46 +0000
@@ -57,6 +57,8 @@
     BugWatchScheduler,
     MAX_SAMPLE_SIZE,
     )
+from lp.buildmaster.enums import BuildQueueStatus
+from lp.buildmaster.model.buildqueue import BuildQueue
 from lp.code.interfaces.revision import IRevisionSet
 from lp.code.model.codeimportevent import CodeImportEvent
 from lp.code.model.codeimportresult import CodeImportResult
@@ -99,6 +101,7 @@
 from lp.services.identity.interfaces.emailaddress import EmailAddressStatus
 from lp.services.identity.model.account import Account
 from lp.services.identity.model.emailaddress import EmailAddress
+from lp.services.job.interfaces.job import JobStatus
 from lp.services.job.model.job import Job
 from lp.services.librarian.model import TimeLimitedToken
 from lp.services.log.logger import PrefixFilter
@@ -1359,6 +1362,41 @@
         transaction.commit()
 
 
+class BuildQueueMigrator(TunableLoop):
+    """Populate the status and build_farm_job columns of BuildQueue."""
+
+    maximum_chunk_size = 5000
+
+    status_map = {
+        JobStatus.WAITING: BuildQueueStatus.WAITING,
+        JobStatus.RUNNING: BuildQueueStatus.RUNNING,
+        JobStatus.SUSPENDED: BuildQueueStatus.SUSPENDED,
+        }
+
+    def __init__(self, log, abort_time=None):
+        super(BuildQueueMigrator, self).__init__(log, abort_time)
+        self.start_at = 1
+        self.store = IMasterStore(BuildQueue)
+
+    def findBuildQueues(self):
+        return self.store.find(
+            BuildQueue, BuildQueue.id >= self.start_at).order_by(BuildQueue.id)
+
+    def isDone(self):
+        return (
+            not getFeatureFlag('buildmaster.buildqueuemigrator.enabled')
+            or self.findBuildQueues().is_empty())
+
+    def __call__(self, chunk_size):
+        bqs = list(self.findBuildQueues()[:chunk_size])
+        for bq in bqs:
+            bq._build_farm_job = bq.specific_job.build.build_farm_job
+            bq._date_started = bq.job.date_started
+            bq.status = self.status_map[bq.job.status]
+        self.start_at = bqs[-1].id + 1
+        transaction.commit()
+
+
 class BaseDatabaseGarbageCollector(LaunchpadCronScript):
     """Abstract base class to run a collection of TunableLoops."""
     script_name = None  # Script name for locking and database user. Override.
@@ -1613,6 +1651,7 @@
         UnusedSessionPruner,
         DuplicateSessionPruner,
         BugHeatUpdater,
+        BuildQueueMigrator,
         ]
     experimental_tunable_loops = []
 

=== modified file 'lib/lp/scripts/tests/test_garbo.py'
--- lib/lp/scripts/tests/test_garbo.py	2013-06-20 05:50:00 +0000
+++ lib/lp/scripts/tests/test_garbo.py	2013-11-20 00:21:46 +0000
@@ -1,5 +1,6 @@
-# Copyright 2009-2013 Canonical Ltd.  This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
+# Copyright 2009-2013 Canonical Ltd.  This software is licensed under
+# the GNU Affero General Public License version 3 (see the file
+# LICENSE).
 
 """Test the database garbage collector."""
 
@@ -28,6 +29,7 @@
     )
 from storm.store import Store
 from testtools.matchers import (
+    ContainsAll,
     Equals,
     GreaterThan,
     )
@@ -41,6 +43,7 @@
     BugNotification,
     BugNotificationRecipient,
     )
+from lp.buildmaster.interfaces.buildqueue import IBuildQueueSet
 from lp.code.bzr import (
     BranchFormat,
     RepositoryFormat,
@@ -1023,6 +1026,36 @@
             "SELECT COUNT(*) FROM BugSummaryJournal").get_one()[0]
         self.assertThat(num_rows, Equals(0))
 
+    def test_BuildQueueMigrator(self):
+        switch_dbuser('testadmin')
+        store = IMasterStore(CommercialSubscription)
+        tbq = self.factory.makeTranslationTemplatesBuild().queueBuild()
+        sbq = self.factory.makeSourcePackageRecipeBuild().queueBuild()
+        bbq = self.factory.makeBinaryPackageBuild().queueBuild()
+        expected = [
+            (bq.id, bq.status.value,
+             removeSecurityProxy(bq)._build_farm_job.id)
+            for bq in (tbq, sbq, bbq)]
+
+        def get_bq_bits():
+            return store.execute(
+                "SELECT id, status, build_farm_job FROM buildqueue").get_all()
+
+        def count_bq_empties():
+            return store.execute(
+                "SELECT COUNT(*) FROM BuildQueue "
+                "WHERE status IS NULL OR build_farm_job IS NULL").get_one()[0]
+        self.assertThat(get_bq_bits(), ContainsAll(expected))
+        store.execute(
+            "UPDATE BuildQueue SET status = NULL, build_farm_job = NULL")
+        self.assertEqual(5, count_bq_empties())
+        IMasterStore(FeatureFlag).add(FeatureFlag(
+            u'default', 0, u'buildmaster.buildqueuemigrator.enabled',
+            u'please'))
+        self.runHourly()
+        self.assertThat(get_bq_bits(), ContainsAll(expected))
+        self.assertEqual(0, count_bq_empties())
+
     def test_VoucherRedeemer(self):
         switch_dbuser('testadmin')