launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #28446
[Merge] ~cjwatson/launchpad:dominator-channel-map into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:dominator-channel-map into launchpad:master.
Commit message:
Support dominating publications by channel
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/422233
This means that, once we start publishing binaries to snap-store-style semantic channels (e.g. "stable", "1.0/candidate", etc.), the dominator will be able to correctly supersede publications that have been replaced with a different version of the same package in the same channel.
I had to prepare for this by adding channel support to various publishing primitives. There's no UI or API support for any of this yet, but I at least needed enough to support the dominator and its tests.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:dominator-channel-map into launchpad:master.
diff --git a/lib/lp/archivepublisher/domination.py b/lib/lp/archivepublisher/domination.py
index 5cc9422..7160a28 100644
--- a/lib/lp/archivepublisher/domination.py
+++ b/lib/lp/archivepublisher/domination.py
@@ -64,6 +64,7 @@ from storm.expr import (
And,
Count,
Desc,
+ Not,
Select,
)
from zope.component import getUtility
@@ -74,6 +75,7 @@ from lp.services.database.constants import UTC_NOW
from lp.services.database.decoratedresultset import DecoratedResultSet
from lp.services.database.interfaces import IStore
from lp.services.database.sqlbase import flush_database_updates
+from lp.services.database.stormexpr import IsDistinctFrom
from lp.services.orderingcheck import OrderingCheck
from lp.soyuz.enums import (
BinaryPackageFormat,
@@ -204,6 +206,31 @@ class GeneralizedPublication:
return sorted(publications, key=cmp_to_key(self.compare), reverse=True)
+class PublicationLocation:
+ """A representation of a publication's location.
+
+ This currently consists of just its name and channel.
+ """
+ def __init__(self, pub, generalization):
+ self.name = generalization.getPackageName(pub)
+ self.channel = pub.channel
+
+ def __str__(self):
+ s = self.name
+ if self.channel is not None:
+ s += " (%s)" % self.channel
+ return s
+
+ def __eq__(self, other):
+ return self.name == other.name and self.channel == other.channel
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ def __hash__(self):
+ return hash((self.name, self.channel))
+
+
def find_live_source_versions(sorted_pubs):
"""Find versions out of Published publications that should stay live.
@@ -336,10 +363,19 @@ def find_live_binary_versions_pass_2(sorted_pubs, cache):
[pub.binarypackagerelease for pub in arch_indep_pubs], ['buildID'])
load_related(SourcePackageRelease, bpbs, ['source_package_release_id'])
+ # XXX cjwatson 2022-05-01: Skip the architecture-specific check for
+ # publications from CI builds for now, until we figure out how to
+ # approximate source package releases for groups of CI builds. We don't
+ # currently expect problematic situations to come up on production; CI
+ # builds are currently only expected to be used in situations where
+ # either we don't build both architecture-specific and
+ # architecture-independent packages, or where tight dependencies between
+ # the two aren't customary.
reprieved_pubs = [
pub
for pub in arch_indep_pubs
- if cache.hasArchSpecificPublications(pub)]
+ if pub.binarypackagerelease.ci_build_id is None and
+ cache.hasArchSpecificPublications(pub)]
return get_binary_versions([latest] + arch_specific_pubs + reprieved_pubs)
@@ -382,9 +418,9 @@ class Dominator:
list we import.
:param sorted_pubs: A list of publications for the same package,
- in the same archive, series, and pocket, all with status
- `PackagePublishingStatus.PUBLISHED`. They must be sorted from
- most current to least current, as would be the result of
+ in the same archive, series, pocket, and channel, all with
+ status `PackagePublishingStatus.PUBLISHED`. They must be sorted
+ from most current to least current, as would be the result of
`generalization.sortPublications`.
:param live_versions: Iterable of versions that are still considered
"live" for this package. For any of these, the latest publication
@@ -458,31 +494,33 @@ class Dominator:
return supersede, keep, delete
def _sortPackages(self, publications, generalization):
- """Partition publications by package name, and sort them.
+ """Partition publications by package location, and sort them.
The publications are sorted from most current to least current,
- as required by `planPackageDomination` etc.
+ as required by `planPackageDomination` etc. Locations are currently
+ (package name, channel).
:param publications: An iterable of `SourcePackagePublishingHistory`
or of `BinaryPackagePublishingHistory`.
:param generalization: A `GeneralizedPublication` helper representing
the kind of publications these are: source or binary.
- :return: A dict mapping each package name to a sorted list of
- publications from `publications`.
+ :return: A dict mapping each package location (package name,
+ channel) to a sorted list of publications from `publications`.
"""
- pubs_by_package = defaultdict(list)
+ pubs_by_location = defaultdict(list)
for pub in publications:
- pubs_by_package[generalization.getPackageName(pub)].append(pub)
+ location = PublicationLocation(pub, generalization)
+ pubs_by_location[location].append(pub)
# Sort the publication lists. This is not an in-place sort, so
# it involves altering the dict while we iterate it. Listify
# the keys so that we can be sure that we're not altering the
# iteration order while iteration is underway.
- for package in list(pubs_by_package.keys()):
- pubs_by_package[package] = generalization.sortPublications(
- pubs_by_package[package])
+ for location in list(pubs_by_location):
+ pubs_by_location[location] = generalization.sortPublications(
+ pubs_by_location[location])
- return pubs_by_package
+ return pubs_by_location
def _setScheduledDeletionDate(self, pub_record):
"""Set the scheduleddeletiondate on a publishing record.
@@ -541,7 +579,10 @@ class Dominator:
BinaryPackagePublishingHistory.binarypackagerelease ==
BinaryPackageRelease.id,
BinaryPackageRelease.build == BinaryPackageBuild.id,
- BinaryPackagePublishingHistory.pocket == pub_record.pocket)
+ BinaryPackagePublishingHistory.pocket == pub_record.pocket,
+ Not(IsDistinctFrom(
+ BinaryPackagePublishingHistory._channel,
+ pub_record._channel)))
# There is at least one non-removed binary to consider
if not considered_binaries.is_empty():
@@ -552,6 +593,7 @@ class Dominator:
SourcePackagePublishingHistory,
distroseries=pub_record.distroseries,
pocket=pub_record.pocket,
+ channel=pub_record.channel,
status=PackagePublishingStatus.PUBLISHED,
archive=self.archive,
sourcepackagerelease=srcpkg_release)
@@ -588,7 +630,8 @@ class Dominator:
]
candidate_binary_names = Select(
BPPH.binarypackagenameID, And(*bpph_location_clauses),
- group_by=BPPH.binarypackagenameID, having=(Count() > 1))
+ group_by=(BPPH.binarypackagenameID, BPPH._channel),
+ having=(Count() > 1))
main_clauses = bpph_location_clauses + [
BPR.id == BPPH.binarypackagereleaseID,
BPR.binarypackagenameID.is_in(candidate_binary_names),
@@ -664,13 +707,13 @@ class Dominator:
bins = self.findBinariesForDomination(distroarchseries, pocket)
sorted_packages = self._sortPackages(bins, generalization)
self.logger.info("Planning domination of binaries...")
- for name, pubs in sorted_packages.items():
- self.logger.debug("Planning domination of %s" % name)
+ for location, pubs in sorted_packages.items():
+ self.logger.debug("Planning domination of %s" % location)
assert len(pubs) > 0, "Dominating zero binaries!"
live_versions = find_live_binary_versions_pass_1(pubs)
plan(pubs, live_versions)
if contains_arch_indep(pubs):
- packages_w_arch_indep.add(name)
+ packages_w_arch_indep.add(location)
execute_plan()
@@ -692,9 +735,10 @@ class Dominator:
bins = self.findBinariesForDomination(distroarchseries, pocket)
sorted_packages = self._sortPackages(bins, generalization)
self.logger.info("Planning domination of binaries...(2nd pass)")
- for name in packages_w_arch_indep.intersection(sorted_packages):
- pubs = sorted_packages[name]
- self.logger.debug("Planning domination of %s" % name)
+ for location in packages_w_arch_indep.intersection(
+ sorted_packages):
+ pubs = sorted_packages[location]
+ self.logger.debug("Planning domination of %s" % location)
assert len(pubs) > 0, "Dominating zero binaries in 2nd pass!"
live_versions = find_live_binary_versions_pass_2(
pubs, reprieve_cache)
@@ -732,7 +776,7 @@ class Dominator:
candidate_source_names = Select(
SPPH.sourcepackagenameID,
And(join_spph_spr(), spph_location_clauses),
- group_by=SPPH.sourcepackagenameID,
+ group_by=(SPPH.sourcepackagenameID, SPPH._channel),
having=(Count() > 1))
# We'll also access the SourcePackageReleases associated with
@@ -769,8 +813,8 @@ class Dominator:
delete = []
self.logger.debug("Dominating sources...")
- for name, pubs in sorted_packages.items():
- self.logger.debug("Dominating %s" % name)
+ for location, pubs in sorted_packages.items():
+ self.logger.debug("Dominating %s" % location)
assert len(pubs) > 0, "Dominating zero sources!"
live_versions = find_live_source_versions(pubs)
cur_supersede, _, cur_delete = self.planPackageDomination(
diff --git a/lib/lp/archivepublisher/tests/test_dominator.py b/lib/lp/archivepublisher/tests/test_dominator.py
old mode 100755
new mode 100644
index 01c56e4..98917b1
--- a/lib/lp/archivepublisher/tests/test_dominator.py
+++ b/lib/lp/archivepublisher/tests/test_dominator.py
@@ -30,7 +30,10 @@ from lp.archivepublisher.publishing import Publisher
from lp.registry.interfaces.pocket import PackagePublishingPocket
from lp.registry.interfaces.series import SeriesStatus
from lp.services.log.logger import DevNullLogger
-from lp.soyuz.enums import PackagePublishingStatus
+from lp.soyuz.enums import (
+ BinaryPackageFormat,
+ PackagePublishingStatus,
+ )
from lp.soyuz.interfaces.publishing import (
IPublishingSet,
ISourcePackagePublishingHistory,
@@ -398,6 +401,51 @@ class TestDominator(TestNativePublishingBase):
for pub in overrides_2:
self.assertEqual(PackagePublishingStatus.PUBLISHED, pub.status)
+ def test_dominate_by_channel(self):
+ # Publications only dominate other publications in the same channel.
+ # (Currently only tested for binary publications.)
+ with lp_dbuser():
+ archive = self.factory.makeArchive()
+ distroseries = self.factory.makeDistroSeries(
+ distribution=archive.distribution)
+ das = self.factory.makeDistroArchSeries(distroseries=distroseries)
+ repository = self.factory.makeGitRepository(
+ target=self.factory.makeDistributionSourcePackage(
+ distribution=archive.distribution))
+ ci_builds = [
+ self.factory.makeCIBuild(
+ git_repository=repository, distro_arch_series=das)
+ for _ in range(3)]
+ bpn = self.factory.makeBinaryPackageName()
+ bprs = [
+ self.factory.makeBinaryPackageRelease(
+ binarypackagename=bpn, version=version, ci_build=ci_build,
+ binpackageformat=BinaryPackageFormat.WHL)
+ for version, ci_build in zip(("1.0", "1.1", "1.2"), ci_builds)]
+ stable_bpphs = [
+ self.factory.makeBinaryPackagePublishingHistory(
+ binarypackagerelease=bpr, archive=archive,
+ distroarchseries=das, status=PackagePublishingStatus.PUBLISHED,
+ pocket=PackagePublishingPocket.RELEASE, channel="stable")
+ for bpr in bprs[:2]]
+ candidate_bpph = self.factory.makeBinaryPackagePublishingHistory(
+ binarypackagerelease=bprs[2], archive=archive,
+ distroarchseries=das, status=PackagePublishingStatus.PUBLISHED,
+ pocket=PackagePublishingPocket.RELEASE, channel="candidate")
+
+ dominator = Dominator(self.logger, archive)
+ dominator.judgeAndDominate(
+ distroseries, PackagePublishingPocket.RELEASE)
+
+ # The older of the two stable publications is superseded, while the
+ # current stable publication and the candidate publication are left
+ # alone.
+ self.checkPublication(
+ stable_bpphs[0], PackagePublishingStatus.SUPERSEDED)
+ self.checkPublications(
+ (stable_bpphs[1], candidate_bpph),
+ PackagePublishingStatus.PUBLISHED)
+
class TestDomination(TestNativePublishingBase):
"""Test overall domination procedure."""
@@ -1315,7 +1363,7 @@ class TestArchSpecificPublicationsCache(TestCaseWithFactory):
return removeSecurityProxy(self.factory.makeSourcePackageRelease())
def makeBPPH(self, spr=None, arch_specific=True, archive=None,
- distroseries=None):
+ distroseries=None, binpackageformat=None, channel=None):
"""Create a `BinaryPackagePublishingHistory`."""
if spr is None:
spr = self.makeSPR()
@@ -1323,12 +1371,13 @@ class TestArchSpecificPublicationsCache(TestCaseWithFactory):
bpb = self.factory.makeBinaryPackageBuild(
source_package_release=spr, distroarchseries=das)
bpr = self.factory.makeBinaryPackageRelease(
- build=bpb, architecturespecific=arch_specific)
+ build=bpb, binpackageformat=binpackageformat,
+ architecturespecific=arch_specific)
return removeSecurityProxy(
self.factory.makeBinaryPackagePublishingHistory(
binarypackagerelease=bpr, archive=archive,
distroarchseries=das, pocket=PackagePublishingPocket.UPDATES,
- status=PackagePublishingStatus.PUBLISHED))
+ status=PackagePublishingStatus.PUBLISHED, channel=channel))
def test_getKey_is_consistent_and_distinguishing(self):
# getKey consistently returns the same key for the same BPPH,
@@ -1351,14 +1400,19 @@ class TestArchSpecificPublicationsCache(TestCaseWithFactory):
spr, arch_specific=False, archive=dependent.archive,
distroseries=dependent.distroseries)
bpph2 = self.makeBPPH(arch_specific=False)
+ bpph3 = self.makeBPPH(
+ arch_specific=False, binpackageformat=BinaryPackageFormat.WHL,
+ channel="edge")
cache = self.makeCache()
self.assertEqual(
- [True, True, False, False],
+ [True, True, False, False, False, False],
[
cache.hasArchSpecificPublications(bpph1),
cache.hasArchSpecificPublications(bpph1),
cache.hasArchSpecificPublications(bpph2),
cache.hasArchSpecificPublications(bpph2),
+ cache.hasArchSpecificPublications(bpph3),
+ cache.hasArchSpecificPublications(bpph3),
])
def test_hasArchSpecificPublications_caches_results(self):
diff --git a/lib/lp/code/interfaces/cibuild.py b/lib/lp/code/interfaces/cibuild.py
index ed9dedf..0827108 100644
--- a/lib/lp/code/interfaces/cibuild.py
+++ b/lib/lp/code/interfaces/cibuild.py
@@ -163,6 +163,14 @@ class ICIBuildView(IPackageBuildView, IPrivacy):
:return: The corresponding `ILibraryFileAlias`.
"""
+ def createBinaryPackageRelease(
+ binarypackagename, version, summary, description, binpackageformat,
+ architecturespecific, installedsize=None, homepage=None):
+ """Create and return a `BinaryPackageRelease` for this CI build.
+
+ The new binary package release will be linked to this build.
+ """
+
class ICIBuildEdit(IBuildFarmJobEdit):
"""`ICIBuild` methods that require launchpad.Edit."""
diff --git a/lib/lp/code/model/cibuild.py b/lib/lp/code/model/cibuild.py
index f2b44a0..24e7e7f 100644
--- a/lib/lp/code/model/cibuild.py
+++ b/lib/lp/code/model/cibuild.py
@@ -81,6 +81,7 @@ from lp.services.macaroons.interfaces import (
)
from lp.services.macaroons.model import MacaroonIssuerBase
from lp.services.propertycache import cachedproperty
+from lp.soyuz.model.binarypackagerelease import BinaryPackageRelease
from lp.soyuz.model.distroarchseries import DistroArchSeries
@@ -452,6 +453,18 @@ class CIBuild(PackageBuildMixin, StormBase):
"""See `IPackageBuild`."""
# We don't currently send any notifications.
+ def createBinaryPackageRelease(
+ self, binarypackagename, version, summary, description,
+ binpackageformat, architecturespecific, installedsize=None,
+ homepage=None):
+ """See `ICIBuild`."""
+ return BinaryPackageRelease(
+ ci_build=self, binarypackagename=binarypackagename,
+ version=version, summary=summary, description=description,
+ binpackageformat=binpackageformat,
+ architecturespecific=architecturespecific,
+ installedsize=installedsize, homepage=homepage)
+
@implementer(ICIBuildSet)
class CIBuildSet(SpecificBuildFarmJobSourceMixin):
diff --git a/lib/lp/code/model/tests/test_cibuild.py b/lib/lp/code/model/tests/test_cibuild.py
index 80558cf..c06038f 100644
--- a/lib/lp/code/model/tests/test_cibuild.py
+++ b/lib/lp/code/model/tests/test_cibuild.py
@@ -17,6 +17,7 @@ import pytz
from storm.locals import Store
from testtools.matchers import (
Equals,
+ Is,
MatchesListwise,
MatchesSetwise,
MatchesStructure,
@@ -65,6 +66,7 @@ from lp.services.log.logger import BufferLogger
from lp.services.macaroons.interfaces import IMacaroonIssuer
from lp.services.macaroons.testing import MacaroonTestMixin
from lp.services.propertycache import clear_property_cache
+from lp.soyuz.enums import BinaryPackageFormat
from lp.testing import (
person_logged_in,
StormStatementRecorder,
@@ -386,6 +388,24 @@ class TestCIBuild(TestCaseWithFactory):
commit_sha1=build.commit_sha1,
ci_build=build))
+ def test_createBinaryPackageRelease(self):
+ build = self.factory.makeCIBuild()
+ bpn = self.factory.makeBinaryPackageName()
+ bpr = build.createBinaryPackageRelease(
+ bpn, "1.0", "test summary", "test description",
+ BinaryPackageFormat.WHL, False, installedsize=1024,
+ homepage="https://example.com/";)
+ self.assertThat(bpr, MatchesStructure(
+ binarypackagename=Equals(bpn),
+ version=Equals("1.0"),
+ summary=Equals("test summary"),
+ description=Equals("test description"),
+ binpackageformat=Equals(BinaryPackageFormat.WHL),
+ architecturespecific=Is(False),
+ installedsize=Equals(1024),
+ homepage=Equals("https://example.com/";),
+ ))
+
class TestCIBuildSet(TestCaseWithFactory):
diff --git a/lib/lp/soyuz/enums.py b/lib/lp/soyuz/enums.py
index d60ea48..9953f43 100644
--- a/lib/lp/soyuz/enums.py
+++ b/lib/lp/soyuz/enums.py
@@ -208,6 +208,13 @@ class BinaryPackageFileType(DBEnumeratedType):
build environment.
""")
+ WHL = DBItem(6, """
+ Python Wheel
+
+ The "wheel" binary package format for Python, originally defined in
+ U{https://peps.python.org/pep-0427/}.
+ """)
+
class BinaryPackageFormat(DBEnumeratedType):
"""Binary Package Format
@@ -251,6 +258,13 @@ class BinaryPackageFormat(DBEnumeratedType):
This is the binary package format used for shipping debug symbols
in Ubuntu and similar distributions.""")
+ WHL = DBItem(6, """
+ Python Wheel
+
+ The "wheel" binary package format for Python, originally defined in
+ U{https://peps.python.org/pep-0427/}.
+ """)
+
class PackageCopyPolicy(DBEnumeratedType):
"""Package copying policy.
diff --git a/lib/lp/soyuz/interfaces/publishing.py b/lib/lp/soyuz/interfaces/publishing.py
index a1cfeab..21a3cac 100644
--- a/lib/lp/soyuz/interfaces/publishing.py
+++ b/lib/lp/soyuz/interfaces/publishing.py
@@ -50,7 +50,6 @@ from zope.schema import (
Date,
Datetime,
Int,
- List,
Text,
TextLine,
)
@@ -269,9 +268,8 @@ class ISourcePackagePublishingHistoryPublic(IPublishingView):
vocabulary=PackagePublishingPocket,
required=True, readonly=True,
))
- channel = List(
- value_type=TextLine(), title=_("Channel"),
- required=False, readonly=False,
+ channel = TextLine(
+ title=_("Channel"), required=False, readonly=False,
description=_(
"The channel into which this entry is published "
"(only for archives published using Artifactory)"))
@@ -700,9 +698,8 @@ class IBinaryPackagePublishingHistoryPublic(IPublishingView):
vocabulary=PackagePublishingPocket,
required=True, readonly=True,
))
- channel = List(
- value_type=TextLine(), title=_("Channel"),
- required=False, readonly=False,
+ channel = TextLine(
+ title=_("Channel"), required=False, readonly=False,
description=_(
"The channel into which this entry is published "
"(only for archives published using Artifactory)"))
@@ -970,7 +967,8 @@ class IPublishingSet(Interface):
def newSourcePublication(archive, sourcepackagerelease, distroseries,
component, section, pocket, ancestor,
create_dsd_job=True, copied_from_archive=None,
- creator=None, sponsor=None, packageupload=None):
+ creator=None, sponsor=None, packageupload=None,
+ channel=None):
"""Create a new `SourcePackagePublishingHistory`.
:param archive: An `IArchive`
diff --git a/lib/lp/soyuz/model/binarypackagerelease.py b/lib/lp/soyuz/model/binarypackagerelease.py
index 25cae48..7134895 100644
--- a/lib/lp/soyuz/model/binarypackagerelease.py
+++ b/lib/lp/soyuz/model/binarypackagerelease.py
@@ -157,6 +157,8 @@ class BinaryPackageRelease(SQLBase):
determined_filetype = BinaryPackageFileType.UDEB
elif file.filename.endswith(".ddeb"):
determined_filetype = BinaryPackageFileType.DDEB
+ elif file.filename.endswith(".whl"):
+ determined_filetype = BinaryPackageFileType.WHL
else:
raise AssertionError(
'Unsupported file type: %s' % file.filename)
diff --git a/lib/lp/soyuz/model/publishing.py b/lib/lp/soyuz/model/publishing.py
index cdab414..2e2cc1f 100644
--- a/lib/lp/soyuz/model/publishing.py
+++ b/lib/lp/soyuz/model/publishing.py
@@ -12,6 +12,7 @@ __all__ = [
from collections import defaultdict
from datetime import datetime
+import json
from operator import (
attrgetter,
itemgetter,
@@ -20,6 +21,7 @@ from pathlib import Path
import sys
import pytz
+from storm.databases.postgres import JSON
from storm.expr import (
And,
Cast,
@@ -31,10 +33,6 @@ from storm.expr import (
Sum,
)
from storm.info import ClassAlias
-from storm.properties import (
- List,
- Unicode,
- )
from storm.store import Store
from storm.zope import IResultSet
from storm.zope.interfaces import ISQLObjectResultSet
@@ -51,6 +49,10 @@ from lp.registry.interfaces.person import validate_public_person
from lp.registry.interfaces.pocket import PackagePublishingPocket
from lp.registry.interfaces.sourcepackage import SourcePackageType
from lp.registry.model.sourcepackagename import SourcePackageName
+from lp.services.channels import (
+ channel_list_to_string,
+ channel_string_to_list,
+ )
from lp.services.database import bulk
from lp.services.database.constants import UTC_NOW
from lp.services.database.datetimecol import UtcDateTimeCol
@@ -267,7 +269,7 @@ class SourcePackagePublishingHistory(SQLBase, ArchivePublisherBase):
pocket = DBEnum(name='pocket', enum=PackagePublishingPocket,
default=PackagePublishingPocket.RELEASE,
allow_none=False)
- channel = List(name="channel", type=Unicode(), allow_none=True)
+ _channel = JSON(name="channel", allow_none=True)
archive = ForeignKey(dbName="archive", foreignKey="Archive", notNull=True)
copied_from_archive = ForeignKey(
dbName="copied_from_archive", foreignKey="Archive", notNull=False)
@@ -315,6 +317,13 @@ class SourcePackagePublishingHistory(SQLBase, ArchivePublisherBase):
self.distroseries.setNewerDistroSeriesVersions([self])
return get_property_cache(self).newer_distroseries_version
+ @property
+ def channel(self):
+ """See `ISourcePackagePublishingHistory`."""
+ if self._channel is None:
+ return None
+ return channel_list_to_string(*self._channel)
+
def getPublishedBinaries(self):
"""See `ISourcePackagePublishingHistory`."""
publishing_set = getUtility(IPublishingSet)
@@ -538,7 +547,8 @@ class SourcePackagePublishingHistory(SQLBase, ArchivePublisherBase):
component=new_component,
section=new_section,
creator=creator,
- archive=self.archive)
+ archive=self.archive,
+ channel=self.channel)
def copyTo(self, distroseries, pocket, archive, override=None,
create_dsd_job=True, creator=None, sponsor=None,
@@ -564,7 +574,8 @@ class SourcePackagePublishingHistory(SQLBase, ArchivePublisherBase):
creator=creator,
sponsor=sponsor,
copied_from_archive=self.archive,
- packageupload=packageupload)
+ packageupload=packageupload,
+ channel=self.channel)
def getStatusSummaryForBuilds(self):
"""See `ISourcePackagePublishingHistory`."""
@@ -685,7 +696,7 @@ class BinaryPackagePublishingHistory(SQLBase, ArchivePublisherBase):
datemadepending = UtcDateTimeCol(default=None)
dateremoved = UtcDateTimeCol(default=None)
pocket = DBEnum(name='pocket', enum=PackagePublishingPocket)
- channel = List(name="channel", type=Unicode(), allow_none=True)
+ _channel = JSON(name="channel", allow_none=True)
archive = ForeignKey(dbName="archive", foreignKey="Archive", notNull=True)
copied_from_archive = ForeignKey(
dbName="copied_from_archive", foreignKey="Archive", notNull=False)
@@ -779,6 +790,13 @@ class BinaryPackagePublishingHistory(SQLBase, ArchivePublisherBase):
distroseries.name,
self.distroarchseries.architecturetag)
+ @property
+ def channel(self):
+ """See `ISourcePackagePublishingHistory`."""
+ if self._channel is None:
+ return None
+ return channel_list_to_string(*self._channel)
+
def getDownloadCount(self):
"""See `IBinaryPackagePublishingHistory`."""
return self.archive.getPackageDownloadTotal(self.binarypackagerelease)
@@ -836,21 +854,26 @@ class BinaryPackagePublishingHistory(SQLBase, ArchivePublisherBase):
dominant.distroarchseries.architecturetag))
dominant_build = dominant.binarypackagerelease.build
- distroarchseries = dominant_build.distro_arch_series
- if logger is not None:
- logger.debug(
- "The %s build of %s has been judged as superseded by the "
- "build of %s. Arch-specific == %s" % (
- distroarchseries.architecturetag,
- self.binarypackagerelease.title,
- dominant_build.source_package_release.title,
- self.binarypackagerelease.architecturespecific))
- # Binary package releases are superseded by the new build,
- # not the new binary package release. This is because
- # there may not *be* a new matching binary package -
- # source packages can change the binaries they build
- # between releases.
- self.supersededby = dominant_build
+ # XXX cjwatson 2022-05-01: We can't currently dominate with CI
+ # builds, since supersededby is a reference to a BPB. Just
+ # leave supersededby unset in that case for now, which isn't
+ # ideal but will work well enough.
+ if dominant_build is not None:
+ distroarchseries = dominant_build.distro_arch_series
+ if logger is not None:
+ logger.debug(
+ "The %s build of %s has been judged as superseded by "
+ "the build of %s. Arch-specific == %s" % (
+ distroarchseries.architecturetag,
+ self.binarypackagerelease.title,
+ dominant_build.source_package_release.title,
+ self.binarypackagerelease.architecturespecific))
+ # Binary package releases are superseded by the new build,
+ # not the new binary package release. This is because
+ # there may not *be* a new matching binary package -
+ # source packages can change the binaries they build
+ # between releases.
+ self.supersededby = dominant_build
debug = getUtility(IPublishingSet).findCorrespondingDDEBPublications(
[self])
@@ -941,7 +964,8 @@ class BinaryPackagePublishingHistory(SQLBase, ArchivePublisherBase):
priority=new_priority,
creator=creator,
archive=debug.archive,
- phased_update_percentage=new_phased_update_percentage)
+ phased_update_percentage=new_phased_update_percentage,
+ _channel=removeSecurityProxy(debug)._channel)
# Append the modified package publishing entry
return BinaryPackagePublishingHistory(
@@ -957,7 +981,8 @@ class BinaryPackagePublishingHistory(SQLBase, ArchivePublisherBase):
priority=new_priority,
archive=self.archive,
creator=creator,
- phased_update_percentage=new_phased_update_percentage)
+ phased_update_percentage=new_phased_update_percentage,
+ _channel=self._channel)
def copyTo(self, distroseries, pocket, archive):
"""See `BinaryPackagePublishingHistory`."""
@@ -1086,10 +1111,12 @@ class PublishingSet:
"""Utilities for manipulating publications in batches."""
def publishBinaries(self, archive, distroseries, pocket, binaries,
- copied_from_archives=None):
+ copied_from_archives=None, channel=None):
"""See `IPublishingSet`."""
if copied_from_archives is None:
copied_from_archives = {}
+ if channel is not None:
+ channel = channel_string_to_list(channel)
# Expand the dict of binaries into a list of tuples including the
# architecture.
if distroseries.distribution != archive.distribution:
@@ -1124,6 +1151,9 @@ class PublishingSet:
BinaryPackageRelease.binarypackagenameID,
BinaryPackageRelease.version),
BinaryPackagePublishingHistory.pocket == pocket,
+ Not(IsDistinctFrom(
+ BinaryPackagePublishingHistory._channel,
+ json.dumps(channel) if channel is not None else None)),
BinaryPackagePublishingHistory.status.is_in(
active_publishing_status),
BinaryPackageRelease.id ==
@@ -1141,12 +1171,13 @@ class PublishingSet:
BPPH = BinaryPackagePublishingHistory
return bulk.create(
(BPPH.archive, BPPH.copied_from_archive,
- BPPH.distroarchseries, BPPH.pocket,
+ BPPH.distroarchseries, BPPH.pocket, BPPH._channel,
BPPH.binarypackagerelease, BPPH.binarypackagename,
+ BPPH._binarypackageformat,
BPPH.component, BPPH.section, BPPH.priority,
BPPH.phased_update_percentage, BPPH.status, BPPH.datecreated),
- [(archive, copied_from_archives.get(bpr), das, pocket, bpr,
- bpr.binarypackagename,
+ [(archive, copied_from_archives.get(bpr), das, pocket, channel,
+ bpr, bpr.binarypackagename, bpr.binpackageformat,
get_component(archive, das.distroseries, component),
section, priority, phased_update_percentage,
PackagePublishingStatus.PENDING, UTC_NOW)
@@ -1156,7 +1187,7 @@ class PublishingSet:
get_objects=True)
def copyBinaries(self, archive, distroseries, pocket, bpphs, policy=None,
- source_override=None):
+ source_override=None, channel=None):
"""See `IPublishingSet`."""
from lp.soyuz.adapters.overrides import BinaryOverride
if distroseries.distribution != archive.distribution:
@@ -1228,13 +1259,14 @@ class PublishingSet:
bpph.binarypackagerelease: bpph.archive for bpph in bpphs}
return self.publishBinaries(
archive, distroseries, pocket, with_overrides,
- copied_from_archives)
+ copied_from_archives, channel=channel)
def newSourcePublication(self, archive, sourcepackagerelease,
distroseries, component, section, pocket,
ancestor=None, create_dsd_job=True,
copied_from_archive=None,
- creator=None, sponsor=None, packageupload=None):
+ creator=None, sponsor=None, packageupload=None,
+ channel=None):
"""See `IPublishingSet`."""
# Circular import.
from lp.registry.model.distributionsourcepackage import (
@@ -1246,6 +1278,14 @@ class PublishingSet:
"Series distribution %s doesn't match archive distribution %s."
% (distroseries.distribution.name, archive.distribution.name))
+ if (sourcepackagerelease.format == SourcePackageType.DPKG and
+ channel is not None):
+ raise AssertionError(
+ "Can't publish dpkg source packages to a channel")
+
+ if channel is not None:
+ channel = channel_string_to_list(channel)
+
pub = SourcePackagePublishingHistory(
distroseries=distroseries,
pocket=pocket,
@@ -1261,7 +1301,8 @@ class PublishingSet:
ancestor=ancestor,
creator=creator,
sponsor=sponsor,
- packageupload=packageupload)
+ packageupload=packageupload,
+ _channel=channel)
DistributionSourcePackage.ensure(pub)
if create_dsd_job and archive == distroseries.main_archive:
diff --git a/lib/lp/soyuz/tests/test_publishing.py b/lib/lp/soyuz/tests/test_publishing.py
index 41cdf25..580df13 100644
--- a/lib/lp/soyuz/tests/test_publishing.py
+++ b/lib/lp/soyuz/tests/test_publishing.py
@@ -36,6 +36,7 @@ from lp.registry.interfaces.sourcepackage import (
SourcePackageUrgency,
)
from lp.registry.interfaces.sourcepackagename import ISourcePackageNameSet
+from lp.services.channels import channel_string_to_list
from lp.services.config import config
from lp.services.database.constants import UTC_NOW
from lp.services.librarian.interfaces import ILibraryFileAliasSet
@@ -212,7 +213,8 @@ class SoyuzTestPublisher:
build_conflicts_indep=None,
dsc_maintainer_rfc822='Foo Bar <foo@xxxxxxx>',
maintainer=None, creator=None, date_uploaded=UTC_NOW,
- spr_only=False, user_defined_fields=None):
+ spr_only=False, user_defined_fields=None,
+ format=SourcePackageType.DPKG, channel=None):
"""Return a mock source publishing record.
if spr_only is specified, the source is not published and the
@@ -238,7 +240,7 @@ class SoyuzTestPublisher:
spr = distroseries.createUploadedSourcePackageRelease(
sourcepackagename=spn,
- format=SourcePackageType.DPKG,
+ format=format,
maintainer=maintainer,
creator=creator,
component=component,
@@ -289,6 +291,8 @@ class SoyuzTestPublisher:
datepublished = UTC_NOW
else:
datepublished = None
+ if channel is not None:
+ channel = channel_string_to_list(channel)
spph = SourcePackagePublishingHistory(
distroseries=distroseries,
@@ -304,7 +308,8 @@ class SoyuzTestPublisher:
scheduleddeletiondate=scheduleddeletiondate,
pocket=pocket,
archive=archive,
- creator=creator)
+ creator=creator,
+ _channel=channel)
return spph
@@ -328,7 +333,8 @@ class SoyuzTestPublisher:
builder=None,
component='main',
phased_update_percentage=None,
- with_debug=False, user_defined_fields=None):
+ with_debug=False, user_defined_fields=None,
+ channel=None):
"""Return a list of binary publishing records."""
if distroseries is None:
distroseries = self.distroseries
@@ -366,7 +372,7 @@ class SoyuzTestPublisher:
pub_binaries += self.publishBinaryInArchive(
binarypackagerelease_ddeb, archive, status,
pocket, scheduleddeletiondate, dateremoved,
- phased_update_percentage)
+ phased_update_percentage, channel=channel)
else:
binarypackagerelease_ddeb = None
@@ -378,7 +384,8 @@ class SoyuzTestPublisher:
user_defined_fields=user_defined_fields)
pub_binaries += self.publishBinaryInArchive(
binarypackagerelease, archive, status, pocket,
- scheduleddeletiondate, dateremoved, phased_update_percentage)
+ scheduleddeletiondate, dateremoved, phased_update_percentage,
+ channel=channel)
published_binaries.extend(pub_binaries)
package_upload = self.addPackageUpload(
archive, distroseries, pocket,
@@ -476,7 +483,7 @@ class SoyuzTestPublisher:
status=PackagePublishingStatus.PENDING,
pocket=PackagePublishingPocket.RELEASE,
scheduleddeletiondate=None, dateremoved=None,
- phased_update_percentage=None):
+ phased_update_percentage=None, channel=None):
"""Return the corresponding BinaryPackagePublishingHistory."""
distroarchseries = binarypackagerelease.build.distro_arch_series
@@ -485,6 +492,8 @@ class SoyuzTestPublisher:
archs = [distroarchseries]
else:
archs = distroarchseries.distroseries.architectures
+ if channel is not None:
+ channel = channel_string_to_list(channel)
pub_binaries = []
for arch in archs:
@@ -502,7 +511,8 @@ class SoyuzTestPublisher:
datecreated=UTC_NOW,
pocket=pocket,
archive=archive,
- phased_update_percentage=phased_update_percentage)
+ phased_update_percentage=phased_update_percentage,
+ _channel=channel)
if status == PackagePublishingStatus.PUBLISHED:
pub.datepublished = UTC_NOW
pub_binaries.append(pub)
@@ -1583,11 +1593,11 @@ class TestPublishBinaries(TestCaseWithFactory):
layer = LaunchpadZopelessLayer
- def makeArgs(self, bprs, distroseries, archive=None):
+ def makeArgs(self, bprs, distroseries, archive=None, channel=None):
"""Create a dict of arguments for publishBinaries."""
if archive is None:
archive = distroseries.main_archive
- return {
+ args = {
'archive': archive,
'distroseries': distroseries,
'pocket': PackagePublishingPocket.BACKPORTS,
@@ -1596,6 +1606,9 @@ class TestPublishBinaries(TestCaseWithFactory):
self.factory.makeSection(),
PackagePublishingPriority.REQUIRED, 50) for bpr in bprs},
}
+ if channel is not None:
+ args['channel'] = channel
+ return args
def test_architecture_dependent(self):
# Architecture-dependent binaries get created as PENDING in the
@@ -1614,8 +1627,8 @@ class TestPublishBinaries(TestCaseWithFactory):
overrides = args['binaries'][bpr]
self.assertEqual(bpr, bpph.binarypackagerelease)
self.assertEqual(
- (args['archive'], target_das, args['pocket']),
- (bpph.archive, bpph.distroarchseries, bpph.pocket))
+ (args['archive'], target_das, args['pocket'], None),
+ (bpph.archive, bpph.distroarchseries, bpph.pocket, bpph.channel))
self.assertEqual(
overrides,
(bpph.component, bpph.section, bpph.priority,
@@ -1670,30 +1683,59 @@ class TestPublishBinaries(TestCaseWithFactory):
args['pocket'] = PackagePublishingPocket.RELEASE
[another_bpph] = getUtility(IPublishingSet).publishBinaries(**args)
+ def test_channel(self):
+ bpr = self.factory.makeBinaryPackageRelease(
+ binpackageformat=BinaryPackageFormat.WHL)
+ target_das = self.factory.makeDistroArchSeries()
+ args = self.makeArgs([bpr], target_das.distroseries, channel="stable")
+ [bpph] = getUtility(IPublishingSet).publishBinaries(**args)
+ self.assertEqual(bpr, bpph.binarypackagerelease)
+ self.assertEqual(
+ (args["archive"], target_das, args["pocket"], args["channel"]),
+ (bpph.archive, bpph.distroarchseries, bpph.pocket, bpph.channel))
+ self.assertEqual(PackagePublishingStatus.PENDING, bpph.status)
+
+ def test_does_not_duplicate_by_channel(self):
+ bpr = self.factory.makeBinaryPackageRelease(
+ binpackageformat=BinaryPackageFormat.WHL)
+ target_das = self.factory.makeDistroArchSeries()
+ args = self.makeArgs([bpr], target_das.distroseries, channel="stable")
+ [bpph] = getUtility(IPublishingSet).publishBinaries(**args)
+ self.assertContentEqual(
+ [], getUtility(IPublishingSet).publishBinaries(**args))
+ args["channel"] = "edge"
+ [another_bpph] = getUtility(IPublishingSet).publishBinaries(**args)
+
class TestChangeOverride(TestNativePublishingBase):
"""Test that changing overrides works."""
def setUpOverride(self, status=SeriesStatus.DEVELOPMENT,
- pocket=PackagePublishingPocket.RELEASE, binary=False,
- ddeb=False, **kwargs):
+ pocket=PackagePublishingPocket.RELEASE, channel=None,
+ binary=False, format=None, ddeb=False, **kwargs):
self.distroseries.status = status
+ get_pub_kwargs = {"pocket": pocket, "channel": channel}
+ if format is not None:
+ get_pub_kwargs["format"] = format
if ddeb:
- pub = self.getPubBinaries(pocket=pocket, with_debug=True)[2]
+ pub = self.getPubBinaries(with_debug=True, **get_pub_kwargs)[2]
self.assertEqual(
BinaryPackageFormat.DDEB,
pub.binarypackagerelease.binpackageformat)
elif binary:
- pub = self.getPubBinaries(pocket=pocket)[0]
+ pub = self.getPubBinaries(**get_pub_kwargs)[0]
else:
- pub = self.getPubSource(pocket=pocket)
+ pub = self.getPubSource(**get_pub_kwargs)
return pub.changeOverride(**kwargs)
def assertCanOverride(self, status=SeriesStatus.DEVELOPMENT,
- pocket=PackagePublishingPocket.RELEASE, **kwargs):
- new_pub = self.setUpOverride(status=status, pocket=pocket, **kwargs)
+ pocket=PackagePublishingPocket.RELEASE, channel=None,
+ **kwargs):
+ new_pub = self.setUpOverride(
+ status=status, pocket=pocket, channel=channel, **kwargs)
self.assertEqual(new_pub.status, PackagePublishingStatus.PENDING)
self.assertEqual(new_pub.pocket, pocket)
+ self.assertEqual(new_pub.channel, channel)
if "new_component" in kwargs:
self.assertEqual(kwargs["new_component"], new_pub.component.name)
if "new_section" in kwargs:
@@ -1784,6 +1826,12 @@ class TestChangeOverride(TestNativePublishingBase):
self.assertCannotOverride(new_component="partner")
self.assertCannotOverride(binary=True, new_component="partner")
+ def test_preserves_channel(self):
+ self.assertCanOverride(
+ binary=True, format=BinaryPackageFormat.WHL, channel="stable",
+ new_component="universe", new_section="misc", new_priority="extra",
+ new_phased_update_percentage=90)
+
class TestPublishingHistoryView(TestCaseWithFactory):
layer = LaunchpadFunctionalLayer
diff --git a/lib/lp/testing/factory.py b/lib/lp/testing/factory.py
index 89925b6..29c6059 100644
--- a/lib/lp/testing/factory.py
+++ b/lib/lp/testing/factory.py
@@ -4029,6 +4029,7 @@ class BareLaunchpadObjectFactory(ObjectFactory):
creator=None,
packageupload=None,
spr_creator=None,
+ channel=None,
**kwargs):
"""Make a `SourcePackagePublishingHistory`.
@@ -4050,6 +4051,7 @@ class BareLaunchpadObjectFactory(ObjectFactory):
is scheduled to be removed.
:param ancestor: The publication ancestor parameter.
:param creator: The publication creator.
+ :param channel: An optional channel to publish into, as a string.
:param **kwargs: All other parameters are passed through to the
makeSourcePackageRelease call if needed.
"""
@@ -4087,7 +4089,7 @@ class BareLaunchpadObjectFactory(ObjectFactory):
archive, sourcepackagerelease, distroseries,
sourcepackagerelease.component, sourcepackagerelease.section,
pocket, ancestor=ancestor, creator=creator,
- packageupload=packageupload)
+ packageupload=packageupload, channel=channel)
naked_spph = removeSecurityProxy(spph)
naked_spph.status = status
@@ -4113,7 +4115,7 @@ class BareLaunchpadObjectFactory(ObjectFactory):
version=None,
architecturespecific=False,
with_debug=False, with_file=False,
- creator=None):
+ creator=None, channel=None):
"""Make a `BinaryPackagePublishingHistory`."""
if distroarchseries is None:
if archive is None:
@@ -4144,7 +4146,10 @@ class BareLaunchpadObjectFactory(ObjectFactory):
if priority is None:
priority = PackagePublishingPriority.OPTIONAL
if binpackageformat is None:
- binpackageformat = BinaryPackageFormat.DEB
+ if binarypackagerelease is not None:
+ binpackageformat = binarypackagerelease.binpackageformat
+ else:
+ binpackageformat = BinaryPackageFormat.DEB
if binarypackagerelease is None:
# Create a new BinaryPackageBuild and BinaryPackageRelease
@@ -4182,7 +4187,8 @@ class BareLaunchpadObjectFactory(ObjectFactory):
archive, distroarchseries.distroseries, pocket,
{binarypackagerelease: (
binarypackagerelease.component, binarypackagerelease.section,
- priority, None)})
+ priority, None)},
+ channel=channel)
for bpph in bpphs:
naked_bpph = removeSecurityProxy(bpph)
naked_bpph.status = status
@@ -4256,7 +4262,7 @@ class BareLaunchpadObjectFactory(ObjectFactory):
libraryfile=library_file, filetype=filetype))
def makeBinaryPackageRelease(self, binarypackagename=None,
- version=None, build=None,
+ version=None, build=None, ci_build=None,
binpackageformat=None, component=None,
section_name=None, priority=None,
architecturespecific=False,
@@ -4270,22 +4276,27 @@ class BareLaunchpadObjectFactory(ObjectFactory):
date_created=None, debug_package=None,
homepage=None):
"""Make a `BinaryPackageRelease`."""
- if build is None:
+ if build is None and ci_build is None:
build = self.makeBinaryPackageBuild()
if binarypackagename is None or isinstance(binarypackagename, str):
binarypackagename = self.getOrMakeBinaryPackageName(
binarypackagename)
- if version is None:
+ if version is None and build is not None:
version = build.source_package_release.version
if binpackageformat is None:
binpackageformat = BinaryPackageFormat.DEB
- if component is None:
+ if component is None and build is not None:
component = build.source_package_release.component
elif isinstance(component, str):
component = getUtility(IComponentSet)[component]
if isinstance(section_name, str):
section_name = self.makeSection(section_name)
- section = section_name or build.source_package_release.section
+ if section_name is not None:
+ section = section_name
+ elif build is not None:
+ section = build.source_package_release.section
+ else:
+ section = None
if priority is None:
priority = PackagePublishingPriority.OPTIONAL
if summary is None:
@@ -4294,18 +4305,35 @@ class BareLaunchpadObjectFactory(ObjectFactory):
description = self.getUniqueString("description")
if installed_size is None:
installed_size = self.getUniqueInteger()
- bpr = build.createBinaryPackageRelease(
- binarypackagename=binarypackagename, version=version,
- binpackageformat=binpackageformat,
- component=component, section=section, priority=priority,
- summary=summary, description=description,
- architecturespecific=architecturespecific,
- shlibdeps=shlibdeps, depends=depends, recommends=recommends,
- suggests=suggests, conflicts=conflicts, replaces=replaces,
- provides=provides, pre_depends=pre_depends,
- enhances=enhances, breaks=breaks, essential=essential,
- installedsize=installed_size, debug_package=debug_package,
- homepage=homepage)
+ kwargs = {
+ "binarypackagename": binarypackagename,
+ "version": version,
+ "binpackageformat": binpackageformat,
+ "summary": summary,
+ "description": description,
+ "architecturespecific": architecturespecific,
+ "installedsize": installed_size,
+ "homepage": homepage,
+ }
+ if build is not None:
+ kwargs.update({
+ "component": component,
+ "section": section,
+ "priority": priority,
+ "shlibdeps": shlibdeps,
+ "depends": depends,
+ "recommends": recommends,
+ "suggests": suggests,
+ "conflicts": conflicts,
+ "replaces": replaces,
+ "provides": provides,
+ "pre_depends": pre_depends,
+ "enhances": enhances,
+ "breaks": breaks,
+ "essential": essential,
+ "debug_package": debug_package,
+ })
+ bpr = (build or ci_build).createBinaryPackageRelease(**kwargs)
if date_created is not None:
removeSecurityProxy(bpr).datecreated = date_created
return bpr
