launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #03590
[Merge] lp:~stevenk/launchpad/generic-overrides into lp:launchpad
Steve Kowalik has proposed merging lp:~stevenk/launchpad/generic-overrides into lp:launchpad.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~stevenk/launchpad/generic-overrides/+merge/60730
Add a generic overrides support class.
In this context, overrides are a description of which component, section and priority a source or binary package should be stored in. This is most used in sync'ing from Debian, where the component in Debian is 'main', but Ubuntu has the source in 'universe'. It also allows further copies or uploads to the same archive/distroseries to respect the previous settings and use them for that upload.
Nothing makes use of it in this branch, but it sets up the ground work. The package copier will make use of this framework in a future branch, and the plan is to have the publisher and the upload processor also make use of this framework.
I have also done two drive-bys in this branch, I moved a test into adapters/tests, and I deleted the test harness stuff from another test.
--
https://code.launchpad.net/~stevenk/launchpad/generic-overrides/+merge/60730
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~stevenk/launchpad/generic-overrides into lp:launchpad.
=== added file 'lib/lp/soyuz/adapters/overrides.py'
--- lib/lp/soyuz/adapters/overrides.py 1970-01-01 00:00:00 +0000
+++ lib/lp/soyuz/adapters/overrides.py 2011-05-12 06:38:28 +0000
@@ -0,0 +1,247 @@
+# Copyright 2011 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Generic Override Policy classes.
+"""
+
+__metaclass__ = type
+
+__all__ = [
+ 'FromExistingOverridePolicy',
+ 'UbuntuOverridePolicy',
+ 'UnknownOverridePolicy',
+ ]
+
+
+from storm.expr import (
+ And,
+ Desc,
+ Or,
+ SQL,
+ )
+
+from canonical.launchpad.components.decoratedresultset import (
+ DecoratedResultSet,
+ )
+from canonical.launchpad.interfaces.lpstorm import IStore
+from lp.registry.model.sourcepackagename import SourcePackageName
+from lp.soyuz.interfaces.publishing import active_publishing_status
+from lp.soyuz.model.binarypackagename import BinaryPackageName
+from lp.soyuz.model.binarypackagerelease import BinaryPackageRelease
+from lp.soyuz.model.component import Component
+from lp.soyuz.model.distroarchseries import DistroArchSeries
+from lp.soyuz.model.publishing import (
+ BinaryPackagePublishingHistory,
+ SourcePackagePublishingHistory,
+ )
+from lp.soyuz.model.section import Section
+from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
+
+
+class BaseOverridePolicy:
+
+ def calculateOverrides(self, archive, distroseries, pocket, sources=None,
+ binaries=None):
+ raise NotImplementedError("Must be implemented by sub-class.")
+
+
+class FromExistingOverridePolicy(BaseOverridePolicy):
+ """Override policy that returns the SPN, component and section for
+ the latest published source publication, or the BPN, DAS, component,
+ section and priority for the latest published binary publication.
+ """
+
+ def findSourceOverrides(self, archive, distroseries, pocket, spns):
+ store = IStore(SourcePackagePublishingHistory)
+ already_published = DecoratedResultSet(
+ store.find(
+ (SQL("""DISTINCT ON (
+ SourcePackageRelease.sourcepackagename) 0 AS ignore"""),
+ SourcePackageRelease.sourcepackagenameID,
+ SourcePackagePublishingHistory.componentID,
+ SourcePackagePublishingHistory.sectionID),
+ SourcePackagePublishingHistory.pocket == pocket,
+ SourcePackagePublishingHistory.archiveID == archive.id,
+ SourcePackagePublishingHistory.distroseriesID ==
+ distroseries.id,
+ SourcePackagePublishingHistory.status.is_in(
+ active_publishing_status),
+ SourcePackageRelease.id ==
+ SourcePackagePublishingHistory.sourcepackagereleaseID,
+ SourcePackageRelease.sourcepackagenameID.is_in(
+ spn.id for spn in spns)).order_by(
+ SourcePackageRelease.sourcepackagenameID,
+ Desc(SourcePackagePublishingHistory.datecreated),
+ Desc(SourcePackagePublishingHistory.id)),
+ source_resolve_ids, pre_iter_hook=source_eager_load)
+ return list(already_published)
+
+ def findBinaryOverrides(self, archive, distroseries, pocket, binaries):
+ store = IStore(BinaryPackagePublishingHistory)
+ expanded = calculate_target_das(distroseries, binaries)
+ candidates = (
+ make_package_condition(archive, das, bpn)
+ for bpn, das in expanded)
+ already_published = DecoratedResultSet(
+ store.find(
+ (SQL("""DISTINCT ON (
+ BinaryPackagePublishingHistory.distroarchseries,
+ BinaryPackageRelease.binarypackagename) 0
+ AS ignore"""),
+ BinaryPackageRelease.binarypackagenameID,
+ BinaryPackagePublishingHistory.distroarchseriesID,
+ BinaryPackagePublishingHistory.componentID,
+ BinaryPackagePublishingHistory.sectionID,
+ BinaryPackagePublishingHistory.priority),
+ BinaryPackagePublishingHistory.pocket == pocket,
+ BinaryPackagePublishingHistory.status.is_in(
+ active_publishing_status),
+ BinaryPackageRelease.id ==
+ BinaryPackagePublishingHistory.binarypackagereleaseID,
+ Or(*candidates)),
+ binary_resolve_ids, pre_iter_hook=binary_eager_load)
+ return list(already_published)
+
+ def calculateOverrides(self, archive, distroseries, pocket,
+ sources=None, binaries=None):
+ if sources is not None and binaries is not None:
+ raise AssertionError(
+ "Can not check for both source and binary overrides "
+ "together.")
+ if sources:
+ return self.findSourceOverrides(
+ archive, distroseries, pocket, sources)
+ if binaries:
+ return self.findBinaryOverrides(
+ archive, distroseries, pocket, binaries)
+
+
+class UnknownOverridePolicy(BaseOverridePolicy):
+ """Override policy that assumes everything passed in doesn't exist, so
+ returns the defaults.
+ """
+
+ def __init__(self):
+ self.default_component = 'universe'
+
+ def calculateOverrides(self, archive, distroseries, pocket,
+ sources=None, binaries=None):
+ if sources is not None and binaries is not None:
+ raise AssertionError(
+ "Can not check for both source and binary overrides "
+ "together.")
+ if sources:
+ store = IStore(SourcePackageName)
+ store.find(SourcePackageName.id.is_in(spn.id for spn in sources))
+ if archive.default_component:
+ self.default_component = archive.default_component
+ overrides = []
+ for source in sources:
+ overrides.append((
+ store.get(SourcePackageName, source.id),
+ self.default_component, None))
+ return overrides
+ if binaries:
+ store = IStore(BinaryPackageName)
+ store.find(
+ BinaryPackageName.id.is_in(bpn.id for bpn, ign in binaries))
+ if archive.default_component:
+ self.default_component = archive.default_component
+ overrides = []
+ for binary, das in calculate_target_das(distroseries, binaries):
+ overrides.append((
+ store.get(BinaryPackageName, binary.id), das,
+ self.default_component, None, None))
+ return overrides
+
+
+class UbuntuOverridePolicy(FromExistingOverridePolicy,
+ UnknownOverridePolicy):
+ """An override policy that incorporates both the from existing policy
+ and the unknown policy.
+ """
+
+ def sourceOverrides(self, archive, distroseries, pocket, sources=None):
+ total = set(sources)
+ overrides = FromExistingOverridePolicy.calculateOverrides(
+ self, archive, distroseries, pocket, sources=sources)
+ existing = set(override[0] for override in overrides)
+ missing = total.difference(existing)
+ if missing:
+ unknown_overrides = UnknownOverridePolicy.calculateOverrides(
+ self, archive, distroseries, pocket, sources=missing)
+ overrides.extend(unknown_overrides)
+ return overrides
+
+ def binaryOverrides(self, archive, distroseries, pocket, binaries=None):
+ total = set(binaries)
+ overrides = FromExistingOverridePolicy.calculateOverrides(
+ self, archive, distroseries, pocket, binaries=binaries)
+ existing = set((
+ overide[0], overide[1].architecturetag)
+ for overide in overrides)
+ missing = total.difference(existing)
+ if missing:
+ unknown_overrides = UnknownOverridePolicy.calculateOverrides(
+ self, archive, distroseries, pocket, binaries=missing)
+ overrides.extend(unknown_overrides)
+ return overrides
+
+ def calculateOverrides(self, archive, distroseries, pocket,
+ sources=None, binaries=None):
+ if sources:
+ return self.sourceOverrides(
+ archive, distroseries, pocket, sources=sources)
+ if binaries:
+ return self.binaryOverrides(
+ archive, distroseries, pocket, binaries=binaries)
+
+
+def calculate_target_das(distroseries, binaries):
+ archs = list(distroseries.enabled_architectures)
+ arch_map = dict((arch.architecturetag, arch) for arch in archs)
+
+ with_das = []
+ for bpn, archtag in binaries:
+ if archtag is not None:
+ with_das.append((bpn, arch_map.get(archtag)))
+ else:
+ with_das.append((bpn, distroseries.nominatedarchindep))
+ return with_das
+
+
+def make_package_condition(archive, das, bpn):
+ return And(
+ BinaryPackagePublishingHistory.archiveID == archive.id,
+ BinaryPackagePublishingHistory.distroarchseriesID == das.id,
+ BinaryPackageRelease.binarypackagenameID ==
+ bpn.id)
+
+
+def source_eager_load(rows):
+ IStore(Component).find(
+ Component, Component.id.is_in(row[2] for row in rows))
+ IStore(Section).find(
+ Section, Section.id.is_in(row[3] for row in rows))
+
+
+def binary_eager_load(rows):
+ IStore(Component).find(
+ Component, Component.id.is_in(row[3] for row in rows))
+ IStore(Section).find(
+ Section, Section.id.is_in(row[4] for row in rows))
+
+
+def source_resolve_ids(row):
+ store = IStore(SourcePackagePublishingHistory)
+ return (
+ store.get(SourcePackageName, row[1]), store.get(Component, row[2]),
+ store.get(Section, row[3]))
+
+
+def binary_resolve_ids(row):
+ store = IStore(BinaryPackagePublishingHistory)
+ return (
+ store.get(BinaryPackageName, row[1]),
+ store.get(DistroArchSeries, row[2]),
+ store.get(Component, row[3]), store.get(Section, row[4]), row[5])
=== renamed file 'lib/lp/soyuz/tests/test_copypolicy.py' => 'lib/lp/soyuz/adapters/tests/test_copypolicy.py'
=== added file 'lib/lp/soyuz/adapters/tests/test_overrides.py'
--- lib/lp/soyuz/adapters/tests/test_overrides.py 1970-01-01 00:00:00 +0000
+++ lib/lp/soyuz/adapters/tests/test_overrides.py 2011-05-12 06:38:28 +0000
@@ -0,0 +1,229 @@
+# Copyright 2011 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Test generic override policy classes."""
+
+from testtools.matchers import Equals
+
+from canonical.testing.layers import LaunchpadZopelessLayer
+from lp.soyuz.adapters.overrides import (
+ FromExistingOverridePolicy,
+ UbuntuOverridePolicy,
+ UnknownOverridePolicy,
+ )
+from lp.soyuz.enums import PackagePublishingStatus
+from lp.testing import (
+ StormStatementRecorder,
+ TestCaseWithFactory,
+ )
+from lp.testing.matchers import HasQueryCount
+
+
+class TestOverrides(TestCaseWithFactory):
+
+ layer = LaunchpadZopelessLayer
+
+ def test_FromExistingOverridePolicy_both(self):
+ # The FromExistingOverridePolicy() can not handle both source and
+ # binary arguments in one call.
+ distroseries = self.factory.makeDistroSeries()
+ pocket = self.factory.getAnyPocket()
+ policy = FromExistingOverridePolicy()
+ self.assertRaises(
+ AssertionError, policy.calculateOverrides,
+ distroseries.main_archive, distroseries, pocket, sources=(),
+ binaries=())
+
+ def test_no_source_overrides(self):
+ # If the spn is not published in the given archive/distroseries, an
+ # empty list is returned.
+ spn = self.factory.makeSourcePackageName()
+ distroseries = self.factory.makeDistroSeries()
+ pocket = self.factory.getAnyPocket()
+ policy = FromExistingOverridePolicy()
+ overrides = policy.calculateOverrides(
+ distroseries.main_archive, distroseries, pocket, sources=(spn,))
+ self.assertEqual([], overrides)
+
+ def test_source_overrides(self):
+ # When the spn is published in the given archive/distroseries, the
+ # overrides for that archive/distroseries are returned.
+ spph = self.factory.makeSourcePackagePublishingHistory()
+ policy = FromExistingOverridePolicy()
+ overrides = policy.calculateOverrides(
+ spph.distroseries.main_archive, spph.distroseries, spph.pocket,
+ sources=(spph.sourcepackagerelease.sourcepackagename,))
+ expected = [(
+ spph.sourcepackagerelease.sourcepackagename,
+ spph.component, spph.section)]
+ self.assertEqual(expected, overrides)
+
+ def test_source_overrides_latest_only_is_returned(self):
+ # When the spn is published multiple times in the given
+ # archive/distroseries, the latest publication's overrides are
+ # returned.
+ spn = self.factory.makeSourcePackageName()
+ distroseries = self.factory.makeDistroSeries()
+ published_spr = self.factory.makeSourcePackageRelease(
+ sourcepackagename=spn)
+ self.factory.makeSourcePackagePublishingHistory(
+ sourcepackagerelease=published_spr, distroseries=distroseries,
+ status=PackagePublishingStatus.PUBLISHED)
+ spr = self.factory.makeSourcePackageRelease(
+ sourcepackagename=spn)
+ spph = self.factory.makeSourcePackagePublishingHistory(
+ sourcepackagerelease=spr, distroseries=distroseries)
+ policy = FromExistingOverridePolicy()
+ overrides = policy.calculateOverrides(
+ distroseries.main_archive, distroseries, spph.pocket,
+ sources=(spn,))
+ self.assertEqual([(spn, spph.component, spph.section)], overrides)
+
+ def test_source_overrides_constant_query_count(self):
+ # The query count is constant, no matter how many sources are
+ # checked.
+ spns = []
+ distroseries = self.factory.makeDistroSeries()
+ pocket = self.factory.getAnyPocket()
+ for i in xrange(10):
+ spph = self.factory.makeSourcePackagePublishingHistory(
+ distroseries=distroseries, archive=distroseries.main_archive,
+ pocket=pocket)
+ spns.append(spph.sourcepackagerelease.sourcepackagename)
+ policy = FromExistingOverridePolicy()
+ with StormStatementRecorder() as recorder:
+ policy.calculateOverrides(
+ spph.distroseries.main_archive, spph.distroseries,
+ spph.pocket, sources=spns)
+ self.assertThat(recorder, HasQueryCount(Equals(2)))
+
+ def test_no_binary_overrides(self):
+ # if the given binary is not published in the given distroarchseries,
+ # an empty list is returned.
+ distroseries = self.factory.makeDistroSeries()
+ das = self.factory.makeDistroArchSeries(distroseries=distroseries)
+ distroseries.nominatedarchindep = das
+ bpn = self.factory.makeBinaryPackageName()
+ pocket = self.factory.getAnyPocket()
+ policy = FromExistingOverridePolicy()
+ overrides = policy.calculateOverrides(
+ distroseries.main_archive, distroseries, pocket,
+ binaries=((bpn, None),))
+ self.assertEqual([], overrides)
+
+ def test_binary_overrides(self):
+ # When a binary is published in the given distroarchseries, the
+ # overrides are returned.
+ bpph = self.factory.makeBinaryPackagePublishingHistory()
+ distroseries = bpph.distroarchseries.distroseries
+ distroseries.nominatedarchindep = bpph.distroarchseries
+ policy = FromExistingOverridePolicy()
+ overrides = policy.calculateOverrides(
+ distroseries.main_archive, distroseries, bpph.pocket,
+ binaries=((bpph.binarypackagerelease.binarypackagename, None),))
+ expected = [(
+ bpph.binarypackagerelease.binarypackagename,
+ bpph.distroarchseries, bpph.component, bpph.section,
+ bpph.priority)]
+ self.assertEqual(expected, overrides)
+
+ def test_binary_overrides_constant_query_count(self):
+ # The query count is constant, no matter how many bpn-das pairs are
+ # checked.
+ bpns = []
+ distroarchseries = self.factory.makeDistroArchSeries()
+ distroseries = distroarchseries.distroseries
+ distroseries.nominatedarchindep = distroarchseries
+ pocket = self.factory.getAnyPocket()
+ for i in xrange(10):
+ bpph = self.factory.makeBinaryPackagePublishingHistory(
+ distroarchseries=distroarchseries,
+ archive=distroseries.main_archive, pocket=pocket)
+ bpns.append((bpph.binarypackagerelease.binarypackagename, None))
+ policy = FromExistingOverridePolicy()
+ with StormStatementRecorder() as recorder:
+ policy.calculateOverrides(
+ distroseries.main_archive, distroseries, pocket,
+ binaries=bpns)
+ self.assertThat(recorder, HasQueryCount(Equals(5)))
+
+ def test_unknown_sources(self):
+ # If the unknown policy is used, it does no checks, just returns the
+ # defaults.
+ spph = self.factory.makeSourcePackagePublishingHistory()
+ policy = UnknownOverridePolicy()
+ overrides = policy.calculateOverrides(
+ spph.distroseries.main_archive, spph.distroseries, spph.pocket,
+ sources=(spph.sourcepackagerelease.sourcepackagename,))
+ expected = [(spph.sourcepackagerelease.sourcepackagename, 'universe',
+ None)]
+ self.assertEqual(expected, overrides)
+
+ def test_unknown_binaries(self):
+ # If the unknown policy is used, it does no checks, just returns the
+ # defaults.
+ bpph = self.factory.makeBinaryPackagePublishingHistory()
+ distroseries = bpph.distroarchseries.distroseries
+ distroseries.nominatedarchindep = bpph.distroarchseries
+ policy = UnknownOverridePolicy()
+ overrides = policy.calculateOverrides(
+ distroseries.main_archive, distroseries, bpph.pocket,
+ binaries=((bpph.binarypackagerelease.binarypackagename, None),))
+ expected = [(bpph.binarypackagerelease.binarypackagename,
+ bpph.distroarchseries, 'universe', None, None)]
+ self.assertEqual(expected, overrides)
+
+ def test_ubuntu_override_policy_sources(self):
+ # The Ubuntu policy incorporates both the existing and the unknown
+ # policy.
+ spns = [self.factory.makeSourcePackageName()]
+ expected = [(spns[0], 'universe', None)]
+ distroseries = self.factory.makeDistroSeries()
+ pocket = self.factory.getAnyPocket()
+ for i in xrange(8):
+ spph = self.factory.makeSourcePackagePublishingHistory(
+ distroseries=distroseries, archive=distroseries.main_archive,
+ pocket=pocket)
+ spns.append(spph.sourcepackagerelease.sourcepackagename)
+ expected.append((
+ spph.sourcepackagerelease.sourcepackagename, spph.component,
+ spph.section))
+ spns.append(self.factory.makeSourcePackageName())
+ expected.append((spns[-1], 'universe', None))
+ policy = UbuntuOverridePolicy()
+ overrides = policy.calculateOverrides(
+ distroseries.main_archive, distroseries, pocket, sources=spns)
+ self.assertEqual(10, len(overrides))
+ self.assertContentEqual(expected, overrides)
+
+ def test_ubuntu_override_policy_binaries(self):
+ # The Ubuntu policy incorporates both the existing and the unknown
+ # policy.
+ distroseries = self.factory.makeDistroSeries()
+ pocket = self.factory.getAnyPocket()
+ bpn = self.factory.makeBinaryPackageName()
+ bpns = []
+ expected = []
+ for i in xrange(3):
+ distroarchseries = self.factory.makeDistroArchSeries(
+ distroseries=distroseries)
+ bpr = self.factory.makeBinaryPackageRelease(
+ binarypackagename=bpn)
+ bpph = self.factory.makeBinaryPackagePublishingHistory(
+ binarypackagerelease=bpr, distroarchseries=distroarchseries,
+ archive=distroseries.main_archive, pocket=pocket)
+ bpns.append((bpn, distroarchseries.architecturetag))
+ expected.append((
+ bpn, distroarchseries, bpph.component, bpph.section,
+ bpph.priority))
+ for i in xrange(2):
+ distroarchseries = self.factory.makeDistroArchSeries(
+ distroseries=distroseries)
+ bpns.append((bpn, distroarchseries.architecturetag))
+ expected.append((bpn, distroarchseries, 'universe', None, None))
+ distroseries.nominatedarchindep = distroarchseries
+ policy = UbuntuOverridePolicy()
+ overrides = policy.calculateOverrides(
+ distroseries.main_archive, distroseries, pocket, binaries=bpns)
+ self.assertEqual(5, len(overrides))
+ self.assertContentEqual(expected, overrides)
=== modified file 'lib/lp/soyuz/adapters/tests/test_packagelocation.py'
--- lib/lp/soyuz/adapters/tests/test_packagelocation.py 2010-10-04 19:50:45 +0000
+++ lib/lp/soyuz/adapters/tests/test_packagelocation.py 2011-05-12 06:38:28 +0000
@@ -3,8 +3,6 @@
"""Test PackageLocation class."""
-import unittest
-
from zope.component import getUtility
from canonical.testing.layers import LaunchpadZopelessLayer
@@ -260,7 +258,3 @@
str(location_ubuntu_packageset),
'Primary Archive for Ubuntu Linux: '
'hoary-RELEASE [foo-packageset]')
-
-
-def test_suite():
- return unittest.TestLoader().loadTestsFromName(__name__)
