launchpad-reviewers team mailing list archive

[William Grant <me@xxxxxxxxxxxxxxxxxx>]

William Grant has proposed merging lp:~wgrant/launchpad/bpb-arch_indep into lp:launchpad with lp:~wgrant/launchpad/bpb-arch_indep-db as a prerequisite.

Commit message:
Model support for BinaryPackageBuild.arch_indep, and a garbo job to backfill it based on nominatedarchindep.

Requested reviews:
  William Grant (wgrant): codew
Related bugs:
  Bug #1350208 in Launchpad itself: "createMissingBuilds misbehaves when copying between series with different nominatedarchindep"
  https://bugs.launchpad.net/launchpad/+bug/1350208

For more details, see:
https://code.launchpad.net/~wgrant/launchpad/bpb-arch_indep/+merge/240400

With vivid switching nominatedarchindep from i386 to amd64, the nominatedarchindep architecture needs to sometimes build arch-indep binaries, and sometimes not. This has to be judged at createMissingBuilds time, which means we need a way to persist it.

This branch starts setting the new column based on nominatedarchindep, and adds a garbo job to backfill existing rows. The garbo job's functionality is replicated by the branch's DB patch 2209-59-1, but for production we need to incrementally do it online.
-- 
https://code.launchpad.net/~wgrant/launchpad/bpb-arch_indep/+merge/240400
Your team Launchpad code reviewers is subscribed to branch lp:launchpad.

=== modified file 'database/schema/security.cfg'
--- database/schema/security.cfg	2014-07-03 04:51:30 +0000
+++ database/schema/security.cfg	2014-11-03 07:32:55 +0000
@@ -2237,6 +2237,7 @@
 public.answercontact                    = SELECT, DELETE
 public.branch                           = SELECT, UPDATE
 public.branchjob                        = SELECT, DELETE
+public.binarypackagebuild               = SELECT, UPDATE
 public.binarypackagename                = SELECT
 public.binarypackagerelease             = SELECT
 public.binarypackagepublishinghistory   = SELECT, UPDATE

=== modified file 'lib/lp/scripts/garbo.py'
--- lib/lp/scripts/garbo.py	2014-06-27 12:50:55 +0000
+++ lib/lp/scripts/garbo.py	2014-11-03 07:32:55 +0000
@@ -117,6 +117,7 @@
 from lp.services.session.model import SessionData
 from lp.services.verification.model.logintoken import LoginToken
 from lp.soyuz.model.archive import Archive
+from lp.soyuz.model.binarypackagebuild import BinaryPackageBuild
 from lp.soyuz.model.livefsbuild import LiveFSFile
 from lp.soyuz.model.publishing import SourcePackagePublishingHistory
 from lp.soyuz.model.reporting import LatestPersonSourcePackageReleaseCache
@@ -1346,6 +1347,36 @@
         transaction.commit()
 
 
+class BinaryPackageBuildArchIndepPopulator(TunableLoop):
+    """Populate the status and build_farm_job columns of BuildQueue."""
+
+    maximum_chunk_size = 5000
+
+    def __init__(self, log, abort_time=None):
+        super(BinaryPackageBuildArchIndepPopulator, self).__init__(
+            log, abort_time)
+        self.start_at = 1
+        self.store = IMasterStore(BinaryPackageBuild)
+
+    def findBuilds(self):
+        return self.store.find(
+            BinaryPackageBuild,
+            BinaryPackageBuild.id >= self.start_at).order_by(
+                BinaryPackageBuild.id)
+
+    def isDone(self):
+        return (
+            not getFeatureFlag('soyuz.bpb_arch_indep_populator.enabled')
+            or self.findBuilds().is_empty())
+
+    def __call__(self, chunk_size):
+        bpbs = list(self.findBuilds()[:chunk_size])
+        for bpb in bpbs:
+            bpb.arch_indep = bpb.distro_arch_series.isNominatedArchIndep
+        self.start_at = bpbs[-1].id + 1
+        transaction.commit()
+
+
 class LiveFSFilePruner(BulkPruner):
     """A BulkPruner to remove old `LiveFSFile`s.
 
@@ -1621,6 +1652,7 @@
         UnusedSessionPruner,
         DuplicateSessionPruner,
         BugHeatUpdater,
+        BinaryPackageBuildArchIndepPopulator,
         ]
     experimental_tunable_loops = []
 

=== modified file 'lib/lp/scripts/tests/test_garbo.py'
--- lib/lp/scripts/tests/test_garbo.py	2014-09-02 02:58:04 +0000
+++ lib/lp/scripts/tests/test_garbo.py	2014-11-03 07:32:55 +0000
@@ -116,7 +116,10 @@
     TestCase,
     TestCaseWithFactory,
     )
-from lp.testing.dbuser import switch_dbuser
+from lp.testing.dbuser import (
+    dbuser,
+    switch_dbuser,
+    )
 from lp.testing.layers import (
     DatabaseLayer,
     LaunchpadScriptLayer,
@@ -1308,6 +1311,23 @@
         self._test_LiveFSFilePruner(
             'application/octet-stream', 0, expected_count=1)
 
+    def test_BinaryPackageBuildArchIndepPopulator(self):
+        with dbuser('testadmin'):
+            ds = self.factory.makeDistroSeries()
+            das1 = self.factory.makeDistroArchSeries(distroseries=ds)
+            ds.nominatedarchindep = das1
+            das2 = self.factory.makeDistroArchSeries(distroseries=ds)
+            bpb1 = self.factory.makeBinaryPackageBuild(distroarchseries=das1)
+            bpb2 = self.factory.makeBinaryPackageBuild(distroarchseries=das2)
+            removeSecurityProxy(bpb1).arch_indep = None
+            removeSecurityProxy(bpb2).arch_indep = None
+
+        with FeatureFixture({'soyuz.bpb_arch_indep_populator.enabled': 'on'}):
+            self.runHourly()
+
+        self.assertIs(True, bpb1.arch_indep)
+        self.assertIs(False, bpb2.arch_indep)
+
 
 class TestGarboTasks(TestCaseWithFactory):
     layer = LaunchpadZopelessLayer

=== modified file 'lib/lp/soyuz/interfaces/binarypackagebuild.py'
--- lib/lp/soyuz/interfaces/binarypackagebuild.py	2014-10-31 07:45:43 +0000
+++ lib/lp/soyuz/interfaces/binarypackagebuild.py	2014-11-03 07:32:55 +0000
@@ -85,6 +85,10 @@
 
     distro_arch_series_id = Int()
 
+    arch_indep = Bool(
+        title=_("Build architecture independent packages"),
+        required=False, readonly=True)
+
     # Properties
     current_source_publication = exported(
         Reference(
@@ -296,13 +300,15 @@
     """Interface for BinaryPackageBuildSet"""
 
     def new(source_package_release, archive, distro_arch_series, pocket,
-            status=BuildStatus.NEEDSBUILD, builder=None):
+            arch_indep=False, status=BuildStatus.NEEDSBUILD, builder=None):
         """Create a new `IBinaryPackageBuild`.
 
         :param source_package_release: An `ISourcePackageRelease`.
         :param archive: An `IArchive` in which context the build is built.
         :param distro_arch_series: An `IDistroArchSeries`.
         :param pocket: An item of `PackagePublishingPocket`.
+        :param arch_indep: Build architecture independent packages in
+            addition to architecture specific ones.
         :param status: A `BuildStatus` item indicating the builds status.
         :param builder: An optional `IBuilder`.
         """

=== modified file 'lib/lp/soyuz/model/binarypackagebuild.py'
--- lib/lp/soyuz/model/binarypackagebuild.py	2014-10-31 08:05:00 +0000
+++ lib/lp/soyuz/model/binarypackagebuild.py	2014-11-03 07:32:55 +0000
@@ -170,6 +170,8 @@
     pocket = DBEnum(
         name='pocket', enum=PackagePublishingPocket, allow_none=False)
 
+    arch_indep = Bool()
+
     upload_log_id = Int(name='upload_log')
     upload_log = Reference(upload_log_id, 'LibraryFileAlias.id')
 
@@ -895,7 +897,7 @@
     implements(IBinaryPackageBuildSet)
 
     def new(self, source_package_release, archive, distro_arch_series, pocket,
-            status=BuildStatus.NEEDSBUILD, builder=None):
+            arch_indep=False, status=BuildStatus.NEEDSBUILD, builder=None):
         """See `IBinaryPackageBuildSet`."""
         # Force the current timestamp instead of the default UTC_NOW for
         # the transaction, avoid several row with same datecreated.
@@ -908,8 +910,8 @@
             build_farm_job=build_farm_job,
             distro_arch_series=distro_arch_series,
             source_package_release=source_package_release,
-            archive=archive, pocket=pocket, status=status,
-            processor=distro_arch_series.processor,
+            archive=archive, pocket=pocket, arch_indep=arch_indep,
+            status=status, processor=distro_arch_series.processor,
             virtualized=archive.require_virtualized, builder=builder,
             is_distro_archive=archive.is_main,
             distribution=distro_arch_series.distroseries.distribution,
@@ -1441,7 +1443,8 @@
 
         build = self.new(
             source_package_release=sourcepackagerelease,
-            distro_arch_series=arch, archive=archive, pocket=pocket)
+            distro_arch_series=arch, archive=archive, pocket=pocket,
+            arch_indep=arch.isNominatedArchIndep)
         # Create the builds in suspended mode for disabled archives.
         build_queue = build.queueBuild(suspended=not archive.enabled)
         Store.of(build).flush()

=== modified file 'lib/lp/soyuz/tests/test_build_set.py'
--- lib/lp/soyuz/tests/test_build_set.py	2014-10-31 08:05:00 +0000
+++ lib/lp/soyuz/tests/test_build_set.py	2014-11-03 07:32:55 +0000
@@ -317,6 +317,17 @@
         self.assertEqual(self.avr_distroarch, builds[0].distro_arch_series)
         self.assertEqual(self.sparc_distroarch, builds[1].distro_arch_series)
 
+    def test_createMissingBuilds_sets_arch_indep(self):
+        """createMissingBuilds() sets arch_indep=True on builds for the
+        nominatedarchindep architecture.
+        """
+        getUtility(IArchiveArchSet).new(self.archive, self.avr)
+        pubrec = self.getPubSource(architecturehintlist='any')
+        builds = pubrec.createMissingBuilds()
+        self.assertContentEqual(
+            [('avr2001', False), ('sparc64', True)],
+            [(build.processor.name, build.arch_indep) for build in builds])
+
 
 class TestFindBySourceAndLocation(TestCaseWithFactory):
     """Tests for the _findBySourceAndLocation helper."""