← Back to team overview

launchpad-reviewers team mailing list archive

[Merge] lp:~wgrant/launchpad/overrides-cross-archs into lp:launchpad

 

William Grant has proposed merging lp:~wgrant/launchpad/overrides-cross-archs into lp:launchpad with lp:~wgrant/launchpad/overrides-include-versions as a prerequisite.

Commit message:
A couple of improvements to FromExistingOverridePolicy: it will filter by pocket if given a pocket, and calculateBinaryOverrides can be asked to ignore the archtag.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~wgrant/launchpad/overrides-cross-archs/+merge/227515

A couple of improvements to FromExistingOverridePolicy: it will filter by pocket if given a pocket, and calculateBinaryOverrides can be asked to ignore the archtag. Both of these changes are needed by archiveuploader.

All IOverridePolicies previously just ignored the pocket argument, and proposed-migration and SRUs rely on taking the most recent series-wide overrides. So I adjusted packagecopier to not ask for a specific pocket; we can revise that later.


-- 
https://code.launchpad.net/~wgrant/launchpad/overrides-cross-archs/+merge/227515
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~wgrant/launchpad/overrides-cross-archs into lp:launchpad.
=== modified file 'lib/lp/soyuz/adapters/overrides.py'
--- lib/lp/soyuz/adapters/overrides.py	2014-07-21 09:10:24 +0000
+++ lib/lp/soyuz/adapters/overrides.py	2014-07-21 09:10:24 +0000
@@ -101,6 +101,7 @@
 
     def __eq__(self, other):
         return (
+            self.__class__ == other.__class__ and
             self.component == other.component and
             self.section == other.section and
             self.version == other.version)
@@ -125,6 +126,7 @@
 
     def __eq__(self, other):
         return (
+            self.__class__ == other.__class__ and
             self.component == other.component and
             self.section == other.section and
             self.priority == other.priority and
@@ -215,7 +217,7 @@
             status.append(PackagePublishingStatus.DELETED)
         return status
 
-    def calculateSourceOverrides(self, archive, distroseries, pockets, sources,
+    def calculateSourceOverrides(self, archive, distroseries, pocket, sources,
                                  include_deleted=False):
         def eager_load(rows):
             bulk.load(Component, (row[1] for row in rows))
@@ -223,6 +225,10 @@
 
         spns = sources.keys()
         store = IStore(SourcePackagePublishingHistory)
+        other_conditions = []
+        if pocket is not None:
+            other_conditions.append(
+                SourcePackagePublishingHistory.pocket == pocket)
         already_published = DecoratedResultSet(
             store.find(
                 (SourcePackagePublishingHistory.sourcepackagenameID,
@@ -237,7 +243,8 @@
                 SourcePackagePublishingHistory.status.is_in(
                     self.getExistingPublishingStatuses(include_deleted)),
                 SourcePackagePublishingHistory.sourcepackagenameID.is_in(
-                    spn.id for spn in spns)).order_by(
+                    spn.id for spn in spns),
+                *other_conditions).order_by(
                         SourcePackagePublishingHistory.sourcepackagenameID,
                         Desc(SourcePackagePublishingHistory.datecreated),
                         Desc(SourcePackagePublishingHistory.id),
@@ -252,19 +259,37 @@
             for (name, component, section, version) in already_published)
 
     def calculateBinaryOverrides(self, archive, distroseries, pocket,
-                                 binaries, include_deleted=False):
+                                 binaries, include_deleted=False,
+                                 any_arch=False):
         def eager_load(rows):
             bulk.load(Component, (row[2] for row in rows))
             bulk.load(Section, (row[3] for row in rows))
 
         store = IStore(BinaryPackagePublishingHistory)
-        expanded = calculate_target_das(distroseries, binaries.keys())
-
-        candidates = [
-            make_package_condition(archive, das, bpn)
-            for bpn, das in expanded if das is not None]
+        other_conditions = []
+        if not any_arch:
+            expanded = calculate_target_das(distroseries, binaries.keys())
+            candidates = [
+                make_package_condition(archive, das, bpn)
+                for bpn, das in expanded if das is not None]
+        else:
+            candidates = []
+            archtags = set()
+            for bpn, archtag in binaries.keys():
+                candidates.append(
+                    BinaryPackagePublishingHistory.binarypackagenameID ==
+                        bpn.id)
+                archtags.add(archtag)
+            other_conditions.extend([
+                DistroArchSeries.distroseriesID == distroseries.id,
+                BinaryPackagePublishingHistory.distroarchseriesID ==
+                    DistroArchSeries.id,
+                ])
         if len(candidates) == 0:
             return {}
+        if pocket is not None:
+            other_conditions.append(
+                BinaryPackagePublishingHistory.pocket == pocket)
         # Do not copy phased_update_percentage from existing publications;
         # it is too context-dependent to copy.
         already_published = DecoratedResultSet(
@@ -279,7 +304,8 @@
                     BinaryPackagePublishingHistory.binarypackagereleaseID,
                 BinaryPackagePublishingHistory.status.is_in(
                     self.getExistingPublishingStatuses(include_deleted)),
-                Or(*candidates)).order_by(
+                Or(*candidates),
+                *other_conditions).order_by(
                     BinaryPackagePublishingHistory.distroarchseriesID,
                     BinaryPackagePublishingHistory.binarypackagenameID,
                     Desc(BinaryPackagePublishingHistory.datecreated),
@@ -297,9 +323,13 @@
         for name, das, component, section, priority, ver in already_published:
             # These details can always fulfill their own archtag, and may
             # satisfy a None archtag if the DAS is nominatedarchindep.
-            matching_keys = [(name, das.architecturetag)]
-            if das == das.distroseries.nominatedarchindep:
-                matching_keys.append((name, None))
+            if not any_arch:
+                matching_keys = [(name, das.architecturetag)]
+                if das == das.distroseries.nominatedarchindep:
+                    matching_keys.append((name, None))
+            else:
+                matching_keys = [
+                    (name, archtag) for archtag in archtags | set((None,))]
             for key in matching_keys:
                 if key not in binaries:
                     continue

=== modified file 'lib/lp/soyuz/adapters/tests/test_overrides.py'
--- lib/lp/soyuz/adapters/tests/test_overrides.py	2014-07-21 09:10:24 +0000
+++ lib/lp/soyuz/adapters/tests/test_overrides.py	2014-07-21 09:10:24 +0000
@@ -30,18 +30,6 @@
 
     layer = ZopelessDatabaseLayer
 
-    def test_no_source_overrides(self):
-        # If the spn is not published in the given archive/distroseries,
-        # no changes are made.
-        spn = self.factory.makeSourcePackageName()
-        distroseries = self.factory.makeDistroSeries()
-        pocket = self.factory.getAnyPocket()
-        policy = FromExistingOverridePolicy()
-        overrides = policy.calculateSourceOverrides(
-            distroseries.main_archive, distroseries, pocket,
-            {spn: SourceOverride()})
-        self.assertEqual({}, overrides)
-
     def test_source_overrides(self):
         # When the spn is published in the given archive/distroseries, the
         # overrides for that archive/distroseries are returned.
@@ -56,6 +44,30 @@
                 version=spph.sourcepackagerelease.version)}
         self.assertEqual(expected, overrides)
 
+    def test_source_overrides_pocket(self):
+        # If the spn is not published in the given pocket, no changes
+        # are made.
+        spn = self.factory.makeSourcePackageName()
+        distroseries = self.factory.makeDistroSeries()
+        policy = FromExistingOverridePolicy()
+        self.factory.makeSourcePackagePublishingHistory(
+            archive=distroseries.main_archive, distroseries=distroseries,
+            pocket=PackagePublishingPocket.RELEASE, sourcepackagename=spn)
+        overrides = policy.calculateSourceOverrides(
+            distroseries.main_archive, distroseries,
+            PackagePublishingPocket.PROPOSED,
+            {spn: SourceOverride()})
+        self.assertEqual(0, len(overrides))
+        overrides = policy.calculateSourceOverrides(
+            distroseries.main_archive, distroseries,
+            PackagePublishingPocket.RELEASE,
+            {spn: SourceOverride()})
+        self.assertEqual(1, len(overrides))
+        overrides = policy.calculateSourceOverrides(
+            distroseries.main_archive, distroseries, None,
+            {spn: SourceOverride()})
+        self.assertEqual(1, len(overrides))
+
     def test_source_overrides_latest_only_is_returned(self):
         # When the spn is published multiple times in the given
         # archive/distroseries, the latest publication's overrides are
@@ -159,6 +171,32 @@
             }
         self.assertEqual(expected, overrides)
 
+    def test_binary_overrides_pocket(self):
+        # If the binary is not published in the given pocket, no changes
+        # are made.
+        distroseries = self.factory.makeDistroSeries()
+        das = self.factory.makeDistroArchSeries(distroseries=distroseries)
+        bpn = self.factory.makeBinaryPackageName()
+        self.factory.makeBinaryPackagePublishingHistory(
+            archive=distroseries.main_archive, distroarchseries=das,
+            pocket=PackagePublishingPocket.RELEASE, binarypackagename=bpn)
+        policy = FromExistingOverridePolicy()
+
+        overrides = policy.calculateBinaryOverrides(
+            distroseries.main_archive, distroseries,
+            PackagePublishingPocket.PROPOSED,
+            {(bpn, das.architecturetag): BinaryOverride()})
+        self.assertEqual(0, len(overrides))
+        overrides = policy.calculateBinaryOverrides(
+            distroseries.main_archive, distroseries,
+            PackagePublishingPocket.RELEASE,
+            {(bpn, das.architecturetag): BinaryOverride()})
+        self.assertEqual(1, len(overrides))
+        overrides = policy.calculateBinaryOverrides(
+            distroseries.main_archive, distroseries, None,
+            {(bpn, das.architecturetag): BinaryOverride()})
+        self.assertEqual(1, len(overrides))
+
     def test_binary_overrides_skips_unknown_arch(self):
         # If calculateBinaryOverrides is passed with an archtag that
         # does not correspond to an ArchSeries of the distroseries,
@@ -176,6 +214,48 @@
             {(bpn, 'i386'): BinaryOverride()})
         self.assertEqual({}, overrides)
 
+    def test_binary_overrides_can_cross_archs(self):
+        # calculateBinaryOverrides can be asked to ignore the archtag
+        # and look for ancestry in any architecture.
+        distroseries = self.factory.makeDistroSeries()
+        amd64 = self.factory.makeDistroArchSeries(
+            architecturetag='amd64',
+            distroseries=distroseries)
+        i386 = self.factory.makeDistroArchSeries(
+            architecturetag='i386',
+            distroseries=distroseries)
+        distroseries.nominatedarchindep = i386
+        bpn = self.factory.makeBinaryPackageName()
+        pocket = self.factory.getAnyPocket()
+        bpph = self.factory.makeBinaryPackagePublishingHistory(
+            archive=distroseries.main_archive, distroarchseries=amd64,
+            pocket=pocket, binarypackagename=bpn, architecturespecific=True)
+        bpph_override = BinaryOverride(
+            component=bpph.component, section=bpph.section,
+            priority=bpph.priority, version=bpph.binarypackagerelease.version)
+        policy = FromExistingOverridePolicy()
+
+        # With any_arch=False only amd64 is found.
+        overrides = policy.calculateBinaryOverrides(
+            distroseries.main_archive, distroseries, pocket,
+            {(bpn, 'i386'): BinaryOverride(),
+             (bpn, 'amd64'): BinaryOverride(),
+             (bpn, None): BinaryOverride()})
+        self.assertEqual({(bpn, 'amd64'): bpph_override}, overrides)
+
+        # But with any_arch=True we get the amd64 overrides everywhere.
+        overrides = policy.calculateBinaryOverrides(
+            distroseries.main_archive, distroseries, pocket,
+            {(bpn, 'i386'): BinaryOverride(),
+             (bpn, 'amd64'): BinaryOverride(),
+             (bpn, None): BinaryOverride()},
+            any_arch=True)
+        self.assertEqual(
+            {(bpn, 'i386'): bpph_override,
+             (bpn, 'amd64'): bpph_override,
+             (bpn, None): bpph_override},
+            overrides)
+
     def test_binary_overrides_constant_query_count(self):
         # The query count is constant, no matter how many bpn-das pairs are
         # checked.

=== modified file 'lib/lp/soyuz/model/packagecopyjob.py'
--- lib/lp/soyuz/model/packagecopyjob.py	2014-07-21 09:10:24 +0000
+++ lib/lp/soyuz/model/packagecopyjob.py	2014-07-21 09:10:24 +0000
@@ -520,8 +520,8 @@
         # runner to suspend the job.
         override_policy = FromExistingOverridePolicy()
         ancestry = override_policy.calculateSourceOverrides(
-            self.target_archive, self.target_distroseries,
-            self.target_pocket, {source_name: SourceOverride()})
+            self.target_archive, self.target_distroseries, None,
+            {source_name: SourceOverride()})
 
         copy_policy = self.getPolicyImplementation()
 
@@ -529,8 +529,7 @@
             # We need to get the default overrides and put them in the
             # metadata.
             defaults = UnknownOverridePolicy().calculateSourceOverrides(
-                self.target_archive, self.target_distroseries,
-                self.target_pocket,
+                self.target_archive, self.target_distroseries, None,
                 {source_name: SourceOverride(component=source_component)})
             self.addSourceOverride(defaults[source_name])
             if auto_approve:

=== modified file 'lib/lp/soyuz/model/publishing.py'
--- lib/lp/soyuz/model/publishing.py	2014-07-21 09:10:24 +0000
+++ lib/lp/soyuz/model/publishing.py	2014-07-21 09:10:24 +0000
@@ -1478,7 +1478,7 @@
                     bpph.distroarchseries.architecturetag)] = bpph
             with_overrides = {}
             overrides = policy.calculateBinaryOverrides(
-                archive, distroseries, pocket,
+                archive, distroseries, None,
                 dict(
                     ((bpn, archtag), BinaryOverride())
                     for bpn, archtag in bpn_archtag.keys()))

=== modified file 'lib/lp/soyuz/scripts/packagecopier.py'
--- lib/lp/soyuz/scripts/packagecopier.py	2014-07-21 09:10:24 +0000
+++ lib/lp/soyuz/scripts/packagecopier.py	2014-07-21 09:10:24 +0000
@@ -721,7 +721,7 @@
             # Only one override can be returned so take the first
             # element of the returned list.
             overrides = policy.calculateSourceOverrides(
-                archive, series, pocket,
+                archive, series, None,
                 {source.sourcepackagerelease.sourcepackagename:
                     SourceOverride()})
             # Only one override can be returned so take the first


Follow ups