launchpad-reviewers team mailing list archive

[William Grant <me@xxxxxxxxxxxxxxxxxx>]

William Grant has proposed merging lp:~wgrant/launchpad/antibfjo-2.5-no-garbo into lp:launchpad with lp:~wgrant/launchpad/antibfjo-2-garbo as a prerequisite.

Commit message:
Drop the BuildQueue backfill garbo job, and migrate sampledata.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~wgrant/launchpad/antibfjo-2.5-no-garbo/+merge/195888

IBuildFarmJobOld and its implementations (BuildPackageJob, SourcePackageRecipeBuildJob, TranslationTemplatesBuildJob) are finally being replaced with an FK from BuildQueue to BuildFarmJob. Additionally, the two fields of Job that are used by BuildQueue are being inlined, and the FK to Job dropped, for performance reasons.

This branch drops the backfilling garbo job, after running it over sampledata.
-- 
https://code.launchpad.net/~wgrant/launchpad/antibfjo-2.5-no-garbo/+merge/195888
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~wgrant/launchpad/antibfjo-2.5-no-garbo into lp:launchpad.

=== modified file 'database/sampledata/current-dev.sql'
--- database/sampledata/current-dev.sql	2013-10-11 11:30:48 +0000
+++ database/sampledata/current-dev.sql	2013-11-20 00:22:10 +0000
@@ -3731,8 +3731,8 @@
 
 ALTER TABLE buildqueue DISABLE TRIGGER ALL;
 
-INSERT INTO buildqueue (id, builder, logtail, lastscore, manual, job, job_type, estimated_duration, processor, virtualized) VALUES (1, 1, 'Dummy sampledata entry, not processing', 1, false, 1, 1, '00:00:00', 1, false);
-INSERT INTO buildqueue (id, builder, logtail, lastscore, manual, job, job_type, estimated_duration, processor, virtualized) VALUES (2, NULL, NULL, 10, false, 2, 1, '00:01:00', 1, false);
+INSERT INTO buildqueue (id, builder, logtail, lastscore, manual, job, job_type, estimated_duration, processor, virtualized, build_farm_job, status, date_started) VALUES (1, 1, 'Dummy sampledata entry, not processing', 1, false, 1, 1, '00:00:00', 1, false, 8, 1, '2005-06-15 09:20:12.820778');
+INSERT INTO buildqueue (id, builder, logtail, lastscore, manual, job, job_type, estimated_duration, processor, virtualized, build_farm_job, status, date_started) VALUES (2, NULL, NULL, 10, false, 2, 1, '00:01:00', 1, false, 11, 0, NULL);
 
 
 ALTER TABLE buildqueue ENABLE TRIGGER ALL;

=== modified file 'database/sampledata/current.sql'
--- database/sampledata/current.sql	2013-10-11 11:30:48 +0000
+++ database/sampledata/current.sql	2013-11-20 00:22:10 +0000
@@ -3665,8 +3665,8 @@
 
 ALTER TABLE buildqueue DISABLE TRIGGER ALL;
 
-INSERT INTO buildqueue (id, builder, logtail, lastscore, manual, job, job_type, estimated_duration, processor, virtualized) VALUES (1, 1, 'Dummy sampledata entry, not processing', 1, false, 1, 1, '00:00:00', 1, false);
-INSERT INTO buildqueue (id, builder, logtail, lastscore, manual, job, job_type, estimated_duration, processor, virtualized) VALUES (2, NULL, NULL, 10, false, 2, 1, '00:01:00', 1, false);
+INSERT INTO buildqueue (id, builder, logtail, lastscore, manual, job, job_type, estimated_duration, processor, virtualized, build_farm_job, status, date_started) VALUES (1, 1, 'Dummy sampledata entry, not processing', 1, false, 1, 1, '00:00:00', 1, false, 8, 1, '2005-06-15 09:20:12.820778');
+INSERT INTO buildqueue (id, builder, logtail, lastscore, manual, job, job_type, estimated_duration, processor, virtualized, build_farm_job, status, date_started) VALUES (2, NULL, NULL, 10, false, 2, 1, '00:01:00', 1, false, 11, 0, NULL);
 
 
 ALTER TABLE buildqueue ENABLE TRIGGER ALL;

=== modified file 'database/schema/security.cfg'
--- database/schema/security.cfg	2013-11-20 00:22:10 +0000
+++ database/schema/security.cfg	2013-10-03 04:37:30 +0000
@@ -2241,7 +2241,6 @@
 public.bugtaskflat                      = SELECT
 public.bugwatch                         = SELECT, UPDATE
 public.bugwatchactivity                 = SELECT, DELETE
-public.buildqueue                       = SELECT, UPDATE
 public.codeimportevent                  = SELECT, DELETE
 public.codeimporteventdata              = SELECT, DELETE
 public.codeimportresult                 = SELECT, DELETE

=== modified file 'lib/lp/buildmaster/tests/test_manager.py'
--- lib/lp/buildmaster/tests/test_manager.py	2013-11-20 00:22:10 +0000
+++ lib/lp/buildmaster/tests/test_manager.py	2013-11-20 00:22:10 +0000
@@ -121,10 +121,8 @@
         self.assertTrue(job is not None)
         self.assertEqual(job.builder, builder)
         self.assertTrue(job.date_started is not None)
+        self.assertEqual(job.status, BuildQueueStatus.RUNNING)
         self.assertEqual(job.job.status, JobStatus.RUNNING)
-        # XXX: Only unconditional until sampledata is migrated.
-        if job.status is not None:
-            self.assertEqual(job.status, BuildQueueStatus.RUNNING)
         build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(job)
         self.assertEqual(build.status, BuildStatus.BUILDING)
         self.assertEqual(job.logtail, logtail)

=== modified file 'lib/lp/scripts/garbo.py'
--- lib/lp/scripts/garbo.py	2013-11-20 00:22:10 +0000
+++ lib/lp/scripts/garbo.py	2013-06-20 05:50:00 +0000
@@ -57,8 +57,6 @@
     BugWatchScheduler,
     MAX_SAMPLE_SIZE,
     )
-from lp.buildmaster.enums import BuildQueueStatus
-from lp.buildmaster.model.buildqueue import BuildQueue
 from lp.code.interfaces.revision import IRevisionSet
 from lp.code.model.codeimportevent import CodeImportEvent
 from lp.code.model.codeimportresult import CodeImportResult
@@ -101,7 +99,6 @@
 from lp.services.identity.interfaces.emailaddress import EmailAddressStatus
 from lp.services.identity.model.account import Account
 from lp.services.identity.model.emailaddress import EmailAddress
-from lp.services.job.interfaces.job import JobStatus
 from lp.services.job.model.job import Job
 from lp.services.librarian.model import TimeLimitedToken
 from lp.services.log.logger import PrefixFilter
@@ -1362,41 +1359,6 @@
         transaction.commit()
 
 
-class BuildQueueMigrator(TunableLoop):
-    """Populate the status and build_farm_job columns of BuildQueue."""
-
-    maximum_chunk_size = 5000
-
-    status_map = {
-        JobStatus.WAITING: BuildQueueStatus.WAITING,
-        JobStatus.RUNNING: BuildQueueStatus.RUNNING,
-        JobStatus.SUSPENDED: BuildQueueStatus.SUSPENDED,
-        }
-
-    def __init__(self, log, abort_time=None):
-        super(BuildQueueMigrator, self).__init__(log, abort_time)
-        self.start_at = 1
-        self.store = IMasterStore(BuildQueue)
-
-    def findBuildQueues(self):
-        return self.store.find(
-            BuildQueue, BuildQueue.id >= self.start_at).order_by(BuildQueue.id)
-
-    def isDone(self):
-        return (
-            not getFeatureFlag('buildmaster.buildqueuemigrator.enabled')
-            or self.findBuildQueues().is_empty())
-
-    def __call__(self, chunk_size):
-        bqs = list(self.findBuildQueues()[:chunk_size])
-        for bq in bqs:
-            bq._build_farm_job = bq.specific_job.build.build_farm_job
-            bq._date_started = bq.job.date_started
-            bq.status = self.status_map[bq.job.status]
-        self.start_at = bqs[-1].id + 1
-        transaction.commit()
-
-
 class BaseDatabaseGarbageCollector(LaunchpadCronScript):
     """Abstract base class to run a collection of TunableLoops."""
     script_name = None  # Script name for locking and database user. Override.
@@ -1651,7 +1613,6 @@
         UnusedSessionPruner,
         DuplicateSessionPruner,
         BugHeatUpdater,
-        BuildQueueMigrator,
         ]
     experimental_tunable_loops = []
 

=== modified file 'lib/lp/scripts/tests/test_garbo.py'
--- lib/lp/scripts/tests/test_garbo.py	2013-11-20 00:22:10 +0000
+++ lib/lp/scripts/tests/test_garbo.py	2013-06-20 05:50:00 +0000
@@ -1,6 +1,5 @@
-# Copyright 2009-2013 Canonical Ltd.  This software is licensed under
-# the GNU Affero General Public License version 3 (see the file
-# LICENSE).
+# Copyright 2009-2013 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Test the database garbage collector."""
 
@@ -29,7 +28,6 @@
     )
 from storm.store import Store
 from testtools.matchers import (
-    ContainsAll,
     Equals,
     GreaterThan,
     )
@@ -43,7 +41,6 @@
     BugNotification,
     BugNotificationRecipient,
     )
-from lp.buildmaster.interfaces.buildqueue import IBuildQueueSet
 from lp.code.bzr import (
     BranchFormat,
     RepositoryFormat,
@@ -1026,36 +1023,6 @@
             "SELECT COUNT(*) FROM BugSummaryJournal").get_one()[0]
         self.assertThat(num_rows, Equals(0))
 
-    def test_BuildQueueMigrator(self):
-        switch_dbuser('testadmin')
-        store = IMasterStore(CommercialSubscription)
-        tbq = self.factory.makeTranslationTemplatesBuild().queueBuild()
-        sbq = self.factory.makeSourcePackageRecipeBuild().queueBuild()
-        bbq = self.factory.makeBinaryPackageBuild().queueBuild()
-        expected = [
-            (bq.id, bq.status.value,
-             removeSecurityProxy(bq)._build_farm_job.id)
-            for bq in (tbq, sbq, bbq)]
-
-        def get_bq_bits():
-            return store.execute(
-                "SELECT id, status, build_farm_job FROM buildqueue").get_all()
-
-        def count_bq_empties():
-            return store.execute(
-                "SELECT COUNT(*) FROM BuildQueue "
-                "WHERE status IS NULL OR build_farm_job IS NULL").get_one()[0]
-        self.assertThat(get_bq_bits(), ContainsAll(expected))
-        store.execute(
-            "UPDATE BuildQueue SET status = NULL, build_farm_job = NULL")
-        self.assertEqual(5, count_bq_empties())
-        IMasterStore(FeatureFlag).add(FeatureFlag(
-            u'default', 0, u'buildmaster.buildqueuemigrator.enabled',
-            u'please'))
-        self.runHourly()
-        self.assertThat(get_bq_bits(), ContainsAll(expected))
-        self.assertEqual(0, count_bq_empties())
-
     def test_VoucherRedeemer(self):
         switch_dbuser('testadmin')