launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #24512
[Merge] ~cjwatson/launchpad:built-using-model into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:built-using-model into launchpad:master.
Commit message:
Parse Built-Using fields from uploaded binaries
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
Related bugs:
Bug #1868558 in Launchpad itself: "Honour Built-Using field"
https://bugs.launchpad.net/launchpad/+bug/1868558
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/381234
These are parsed into a form that we'll later be able to use in the dominator, with the goal of keeping source publications that are referenced by Built-Using fields in active binary publications.
The most complicated bit is working out which source package releases a given Built-Using field refers to, since there are edge cases where just the name and version can be ambiguous. We do the best we can by following the archive dependencies of the build and finding sources that it could plausibly have used.
Unresolvable entries in Built-Using, including ones that refer to deleted source publications, will result in binary uploads being rejected.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:built-using-model into launchpad:master.
diff --git a/lib/lp/archivepublisher/indices.py b/lib/lp/archivepublisher/indices.py
index 9ea0e9a..6d6d234 100644
--- a/lib/lp/archivepublisher/indices.py
+++ b/lib/lp/archivepublisher/indices.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2014 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
__all__ = [
@@ -212,6 +212,13 @@ def build_binary_stanza_fields(bpr, component, section, priority,
fields.append('Pre-Depends', bpr.pre_depends)
fields.append('Enhances', bpr.enhances)
fields.append('Breaks', bpr.breaks)
+ # Add Built-Using at this position, but conditionally since it may be in
+ # bpr.user_defined_fields instead (for older BPRs or for gina-imported
+ # BPRs with unresolvable Built-Using), and in that case if we were to
+ # add it to IndexStanzaFields now then the one in user_defined_fields
+ # would be ignored.
+ if bpr.built_using:
+ fields.append('Built-Using', bpr.built_using)
fields.append('Essential', essential)
fields.append('Filename', bin_filepath)
fields.append('Size', bin_size)
diff --git a/lib/lp/archivepublisher/tests/test_indices.py b/lib/lp/archivepublisher/tests/test_indices.py
index b5e68f5..3ef5610 100644
--- a/lib/lp/archivepublisher/tests/test_indices.py
+++ b/lib/lp/archivepublisher/tests/test_indices.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Test native archive index generation for Soyuz."""
@@ -144,11 +144,12 @@ class TestNativeArchiveIndexes(TestNativePublishingBase):
See also testSourceStanza, it must present something similar for
binary packages.
"""
+ self.getPubSource(sourcename='built-using', version='123')
pub_binaries = self.getPubBinaries(
depends='biscuit', recommends='foo-dev', suggests='pyfoo',
conflicts='old-foo', replaces='old-foo', provides='foo-master',
pre_depends='master-foo', enhances='foo-super', breaks='old-foo',
- phased_update_percentage=50)
+ built_using='built-using (= 123)', phased_update_percentage=50)
pub_binary = pub_binaries[0]
self.assertEqual(
['Package: foo-bin',
@@ -168,6 +169,7 @@ class TestNativeArchiveIndexes(TestNativePublishingBase):
'Pre-Depends: master-foo',
'Enhances: foo-super',
'Breaks: old-foo',
+ 'Built-Using: built-using (= 123)',
'Filename: pool/main/f/foo/foo-bin_666_all.deb',
'Size: 18',
'MD5sum: ' + self.deb_md5,
@@ -184,10 +186,12 @@ class TestNativeArchiveIndexes(TestNativePublishingBase):
custom fields (Python-Version).
"""
+ self.getPubSource(sourcename='built-using', version='123')
pub_binaries = self.getPubBinaries(
depends='biscuit', recommends='foo-dev', suggests='pyfoo',
conflicts='old-foo', replaces='old-foo', provides='foo-master',
pre_depends='master-foo', enhances='foo-super', breaks='old-foo',
+ built_using='built-using (= 123)',
user_defined_fields=[("Python-Version", ">= 2.4")])
pub_binary = pub_binaries[0]
self.assertEqual(
@@ -208,6 +212,7 @@ class TestNativeArchiveIndexes(TestNativePublishingBase):
'Pre-Depends: master-foo',
'Enhances: foo-super',
'Breaks: old-foo',
+ 'Built-Using: built-using (= 123)',
'Filename: pool/main/f/foo/foo-bin_666_all.deb',
'Size: 18',
'MD5sum: ' + self.deb_md5,
diff --git a/lib/lp/archiveuploader/nascentuploadfile.py b/lib/lp/archiveuploader/nascentuploadfile.py
index aa0ec95..ec6275f 100644
--- a/lib/lp/archiveuploader/nascentuploadfile.py
+++ b/lib/lp/archiveuploader/nascentuploadfile.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Specific models for uploaded files"""
@@ -445,6 +445,7 @@ class BaseBinaryUploadFile(PackageUploadFile):
"Provides",
"Pre-Depends",
"Enhances",
+ "Built-Using",
"Essential",
"Description",
"Installed-Size",
@@ -920,6 +921,7 @@ class BaseBinaryUploadFile(PackageUploadFile):
pre_depends=encoded.get('Pre-Depends', ''),
enhances=encoded.get('Enhances', ''),
breaks=encoded.get('Breaks', ''),
+ built_using=encoded.get('Built-Using', ''),
homepage=encoded.get('Homepage'),
essential=is_essential,
installedsize=installedsize,
diff --git a/lib/lp/registry/model/distroseries.py b/lib/lp/registry/model/distroseries.py
index 51dbeea..4f9185e 100644
--- a/lib/lp/registry/model/distroseries.py
+++ b/lib/lp/registry/model/distroseries.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Database classes for a distribution series."""
@@ -124,6 +124,7 @@ from lp.services.propertycache import (
from lp.services.worlddata.model.language import Language
from lp.soyuz.enums import (
ArchivePurpose,
+ BinarySourceReferenceType,
IndexCompressionType,
PackagePublishingStatus,
PackageUploadStatus,
@@ -1138,6 +1139,9 @@ class DistroSeries(SQLBase, BugTargetBase, HasSpecificationsMixin,
def getBinaryPackagePublishing(self, archtag, pocket, component, archive):
"""See `IDistroSeries`."""
+ # Circular import.
+ from lp.soyuz.model.binarysourcereference import BinarySourceReference
+
bpphs = Store.of(self).find(
BinaryPackagePublishingHistory,
DistroArchSeries.distroseries == self,
@@ -1161,6 +1165,20 @@ class DistroSeries(SQLBase, BugTargetBase, HasSpecificationsMixin,
bpbs = load_related(BinaryPackageBuild, bprs, ["buildID"])
sprs = load_related(
SourcePackageRelease, bpbs, ["source_package_release_id"])
+ built_using_bsrs = load_referencing(
+ BinarySourceReference, bprs, ["binary_package_release_id"],
+ extra_conditions=[
+ BinarySourceReference.reference_type ==
+ BinarySourceReferenceType.BUILT_USING,
+ ])
+ # Make sure this is initialised for all BPRs, as some may not
+ # have any BinarySourceReferences.
+ built_using_bsr_map = {bpr: [] for bpr in bprs}
+ for bsr in built_using_bsrs:
+ built_using_bsr_map[bsr.binary_package_release].append(bsr)
+ for bpr, bsrs in built_using_bsr_map.items():
+ get_property_cache(bpr).built_using_references = sorted(
+ bsrs, key=attrgetter("id"))
bpfs = load_referencing(
BinaryPackageFile, bprs, ["binarypackagereleaseID"])
file_map = collections.defaultdict(list)
diff --git a/lib/lp/soyuz/configure.zcml b/lib/lp/soyuz/configure.zcml
index 89c9d83..2e97958 100644
--- a/lib/lp/soyuz/configure.zcml
+++ b/lib/lp/soyuz/configure.zcml
@@ -70,6 +70,20 @@
set_attributes="changelog"/>
</class>
+ <!-- BinarySourceReference -->
+
+ <class
+ class="lp.soyuz.model.binarysourcereference.BinarySourceReference">
+ <allow
+ interface="lp.soyuz.interfaces.binarysourcereference.IBinarySourceReference"/>
+ </class>
+ <securedutility
+ class="lp.soyuz.model.binarysourcereference.BinarySourceReferenceSet"
+ provides="lp.soyuz.interfaces.binarysourcereference.IBinarySourceReferenceSet">
+ <allow
+ interface="lp.soyuz.interfaces.binarysourcereference.IBinarySourceReferenceSet"/>
+ </securedutility>
+
<!-- SourcePackagePublishingHistory -->
<class
diff --git a/lib/lp/soyuz/doc/distroarchseriesbinarypackage.txt b/lib/lp/soyuz/doc/distroarchseriesbinarypackage.txt
index 03d690a..c759bbf 100644
--- a/lib/lp/soyuz/doc/distroarchseriesbinarypackage.txt
+++ b/lib/lp/soyuz/doc/distroarchseriesbinarypackage.txt
@@ -62,6 +62,7 @@ needs to be removed.
... pre_depends=None,
... enhances=None,
... breaks=None,
+ ... built_using=None,
... essential=False,
... installedsize=0,
... architecturespecific=False,
@@ -106,6 +107,7 @@ needs to be removed.
... pre_depends=None,
... enhances=None,
... breaks=None,
+ ... built_using=None,
... essential=False,
... installedsize=0,
... architecturespecific=False,
diff --git a/lib/lp/soyuz/enums.py b/lib/lp/soyuz/enums.py
index b8dafa8..641df6b 100644
--- a/lib/lp/soyuz/enums.py
+++ b/lib/lp/soyuz/enums.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2019 Canonical Ltd. This software is licensed under the
+# Copyright 2010-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Enumerations used in the lp/soyuz modules."""
@@ -13,6 +13,7 @@ __all__ = [
'archive_suffixes',
'BinaryPackageFileType',
'BinaryPackageFormat',
+ 'BinarySourceReferenceType',
'DistroArchSeriesFilterSense',
'IndexCompressionType',
'PackageCopyPolicy',
@@ -613,3 +614,15 @@ class DistroArchSeriesFilterSense(DBEnumeratedType):
Packages in this package set are excluded from the distro arch series.
""")
+
+
+class BinarySourceReferenceType(DBEnumeratedType):
+ """The type of a reference from a binary package to a source package."""
+
+ BUILT_USING = DBItem(1, """
+ Built-Using
+
+ The referencing binary package incorporates part of the referenced
+ source package, and so the referenced source package needs to remain
+ in the archive for as long as the referencing binary package does.
+ """)
diff --git a/lib/lp/soyuz/interfaces/binarypackagebuild.py b/lib/lp/soyuz/interfaces/binarypackagebuild.py
index 8546a85..d811c6c 100644
--- a/lib/lp/soyuz/interfaces/binarypackagebuild.py
+++ b/lib/lp/soyuz/interfaces/binarypackagebuild.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""BinaryPackageBuild interfaces."""
@@ -179,8 +179,8 @@ class IBinaryPackageBuildView(IPackageBuild):
component, section, priority, installedsize, architecturespecific,
shlibdeps=None, depends=None, recommends=None, suggests=None,
conflicts=None, replaces=None, provides=None, pre_depends=None,
- enhances=None, breaks=None, essential=False, debug_package=None,
- user_defined_fields=None, homepage=None):
+ enhances=None, breaks=None, built_using=None, essential=False,
+ debug_package=None, user_defined_fields=None, homepage=None):
"""Create and return a `BinaryPackageRelease`.
The binarypackagerelease will be attached to this specific build.
diff --git a/lib/lp/soyuz/interfaces/binarypackagerelease.py b/lib/lp/soyuz/interfaces/binarypackagerelease.py
index 32fe93c..73f636d 100644
--- a/lib/lp/soyuz/interfaces/binarypackagerelease.py
+++ b/lib/lp/soyuz/interfaces/binarypackagerelease.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2012 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Binary package release interfaces."""
@@ -61,6 +61,9 @@ class IBinaryPackageRelease(Interface):
pre_depends = TextLine(required=False)
enhances = TextLine(required=False)
breaks = TextLine(required=False)
+ built_using_references = List(
+ title=_("Sequence of Built-Using references."), required=True)
+ built_using = TextLine(required=False)
essential = Bool(required=False)
installedsize = Int(required=False)
architecturespecific = Bool(required=True)
diff --git a/lib/lp/soyuz/interfaces/binarysourcereference.py b/lib/lp/soyuz/interfaces/binarysourcereference.py
new file mode 100644
index 0000000..5affa16
--- /dev/null
+++ b/lib/lp/soyuz/interfaces/binarysourcereference.py
@@ -0,0 +1,81 @@
+# Copyright 2020 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Interface for references from binary packages to source packages."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+ 'IBinarySourceReference',
+ 'IBinarySourceReferenceSet',
+ 'UnparsableBuiltUsing',
+ ]
+
+from lazr.restful.fields import Reference
+from zope.interface import Interface
+from zope.schema import (
+ Choice,
+ Int,
+ )
+
+from lp import _
+from lp.soyuz.enums import BinarySourceReferenceType
+from lp.soyuz.interfaces.binarypackagerelease import IBinaryPackageRelease
+from lp.soyuz.interfaces.sourcepackagerelease import ISourcePackageRelease
+
+
+class UnparsableBuiltUsing(Exception):
+ """A Built-Using field could not be parsed."""
+
+
+class IBinarySourceReference(Interface):
+ """A reference from a binary package to a source package."""
+
+ id = Int(title=_("ID"))
+
+ binary_package_release = Reference(
+ IBinaryPackageRelease,
+ title=_("The referencing binary package release."),
+ required=True, readonly=True)
+ source_package_release = Reference(
+ ISourcePackageRelease,
+ title=_("The referenced source package release."),
+ required=True, readonly=True)
+ reference_type = Choice(
+ title=_("The type of the reference."),
+ vocabulary=BinarySourceReferenceType,
+ required=True, readonly=True)
+
+
+class IBinarySourceReferenceSet(Interface):
+ """A set of references from binary packages to source packages."""
+
+ def createFromRelationship(bpr, relationship, reference_type):
+ """Create references from a text relationship field.
+
+ :param bpr: The `IBinaryPackageRelease` from which new references
+ should be created.
+ :param relationship: A text relationship field containing one or
+ more source package relations in the usual Debian encoding (e.g.
+ "source1 (= 1.0), source2 (= 2.0)").
+ :param reference_type: The `BinarySourceReferenceType` of references
+ to create.
+ :return: A list of new `IBinarySourceReference`s.
+ """
+
+ def makeRelationship(references):
+ """Make a text relationship field from some references.
+
+ :param references: An iterable of `IBinarySourceReference`s.
+ :return: A text relationship field in the usual Debian encoding
+ (e.g. "source1 (= 1.0), source2 (= 2.0)").
+ """
+
+ def findByBinaryPackageRelease(bpr, reference_type):
+ """Find references from a given binary package release.
+
+ :param bpr: An `IBinaryPackageRelease` to search for.
+ :param reference_type: A `BinarySourceReferenceType` to search for.
+ :return: A `ResultSet` of matching `IBinarySourceReference`s.
+ """
diff --git a/lib/lp/soyuz/model/binarypackagebuild.py b/lib/lp/soyuz/model/binarypackagebuild.py
index ae4f1de..470a8f2 100644
--- a/lib/lp/soyuz/model/binarypackagebuild.py
+++ b/lib/lp/soyuz/model/binarypackagebuild.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
__metaclass__ = type
@@ -91,6 +91,7 @@ from lp.services.macaroons.model import MacaroonIssuerBase
from lp.soyuz.adapters.buildarch import determine_architectures_to_build
from lp.soyuz.enums import (
ArchivePurpose,
+ BinarySourceReferenceType,
PackagePublishingStatus,
)
from lp.soyuz.interfaces.archive import (
@@ -104,6 +105,9 @@ from lp.soyuz.interfaces.binarypackagebuild import (
IBinaryPackageBuildSet,
UnparsableDependencies,
)
+from lp.soyuz.interfaces.binarysourcereference import (
+ IBinarySourceReferenceSet,
+ )
from lp.soyuz.interfaces.distroarchseries import IDistroArchSeries
from lp.soyuz.interfaces.packageset import IPackagesetSet
from lp.soyuz.mail.binarypackagebuild import BinaryPackageBuildMailer
@@ -662,10 +666,11 @@ class BinaryPackageBuild(PackageBuildMixin, SQLBase):
binpackageformat, component, section, priority, installedsize,
architecturespecific, shlibdeps=None, depends=None, recommends=None,
suggests=None, conflicts=None, replaces=None, provides=None,
- pre_depends=None, enhances=None, breaks=None, essential=False,
- debug_package=None, user_defined_fields=None, homepage=None):
+ pre_depends=None, enhances=None, breaks=None, built_using=None,
+ essential=False, debug_package=None, user_defined_fields=None,
+ homepage=None):
"""See IBuild."""
- return BinaryPackageRelease(
+ bpr = BinaryPackageRelease(
build=self, binarypackagename=binarypackagename, version=version,
summary=summary, description=description,
binpackageformat=binpackageformat,
@@ -677,6 +682,10 @@ class BinaryPackageBuild(PackageBuildMixin, SQLBase):
architecturespecific=architecturespecific,
debug_package=debug_package,
user_defined_fields=user_defined_fields, homepage=homepage)
+ if built_using:
+ getUtility(IBinarySourceReferenceSet).createFromRelationship(
+ bpr, built_using, BinarySourceReferenceType.BUILT_USING)
+ return bpr
def estimateDuration(self):
"""See `IPackageBuild`."""
diff --git a/lib/lp/soyuz/model/binarypackagerelease.py b/lib/lp/soyuz/model/binarypackagerelease.py
index ad4f514..e000483 100644
--- a/lib/lp/soyuz/model/binarypackagerelease.py
+++ b/lib/lp/soyuz/model/binarypackagerelease.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2012 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
__metaclass__ = type
@@ -7,6 +7,7 @@ __all__ = [
'BinaryPackageReleaseDownloadCount',
]
+from operator import attrgetter
import simplejson
from sqlobject import (
@@ -22,6 +23,7 @@ from storm.locals import (
Store,
Storm,
)
+from zope.component import getUtility
from zope.interface import implementer
from lp.services.database.constants import UTC_NOW
@@ -35,12 +37,16 @@ from lp.services.propertycache import (
from lp.soyuz.enums import (
BinaryPackageFileType,
BinaryPackageFormat,
+ BinarySourceReferenceType,
PackagePublishingPriority,
)
from lp.soyuz.interfaces.binarypackagerelease import (
IBinaryPackageRelease,
IBinaryPackageReleaseDownloadCount,
)
+from lp.soyuz.interfaces.binarysourcereference import (
+ IBinarySourceReferenceSet,
+ )
from lp.soyuz.model.files import BinaryPackageFile
@@ -89,6 +95,19 @@ class BinaryPackageRelease(SQLBase):
del kwargs['user_defined_fields']
super(BinaryPackageRelease, self).__init__(*args, **kwargs)
+ @cachedproperty
+ def built_using_references(self):
+ reference_set = getUtility(IBinarySourceReferenceSet)
+ references = reference_set.findByBinaryPackageRelease(
+ self, BinarySourceReferenceType.BUILT_USING)
+ # Preserving insertion order is good enough.
+ return sorted(references, key=attrgetter('id'))
+
+ @property
+ def built_using(self):
+ return getUtility(IBinarySourceReferenceSet).makeRelationship(
+ self.built_using_references)
+
@property
def user_defined_fields(self):
"""See `IBinaryPackageRelease`."""
diff --git a/lib/lp/soyuz/model/binarysourcereference.py b/lib/lp/soyuz/model/binarysourcereference.py
new file mode 100644
index 0000000..1090c1e
--- /dev/null
+++ b/lib/lp/soyuz/model/binarysourcereference.py
@@ -0,0 +1,189 @@
+# Copyright 2020 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""References from binary packages to source packages."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+ 'BinarySourceReference',
+ 'BinarySourceReferenceSet',
+ ]
+
+import warnings
+
+from debian.deb822 import PkgRelation
+from storm.expr import (
+ Column,
+ Table,
+ )
+from storm.locals import (
+ And,
+ Int,
+ Join,
+ Reference,
+ )
+from zope.interface import implementer
+
+from lp.registry.model.sourcepackagename import SourcePackageName
+from lp.services.database.bulk import (
+ create,
+ dbify_value,
+ )
+from lp.services.database.enumcol import DBEnum
+from lp.services.database.interfaces import IStore
+from lp.services.database.stormbase import StormBase
+from lp.services.database.stormexpr import Values
+from lp.soyuz.adapters.archivedependencies import expand_dependencies
+from lp.soyuz.enums import (
+ BinarySourceReferenceType,
+ PackagePublishingStatus,
+ )
+from lp.soyuz.interfaces.binarysourcereference import (
+ IBinarySourceReference,
+ IBinarySourceReferenceSet,
+ UnparsableBuiltUsing,
+ )
+from lp.soyuz.model.publishing import SourcePackagePublishingHistory
+from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
+
+
+@implementer(IBinarySourceReference)
+class BinarySourceReference(StormBase):
+ """See `IBinarySourceReference`."""
+
+ __storm_table__ = "BinarySourceReference"
+
+ id = Int(primary=True)
+
+ binary_package_release_id = Int(
+ name="binary_package_release", allow_none=False)
+ binary_package_release = Reference(
+ binary_package_release_id, "BinaryPackageRelease.id")
+
+ source_package_release_id = Int(
+ name="source_package_release", allow_none=False)
+ source_package_release = Reference(
+ source_package_release_id, "SourcePackageRelease.id")
+
+ reference_type = DBEnum(enum=BinarySourceReferenceType, allow_none=False)
+
+ def __init__(self, binary_package_release, source_package_release,
+ reference_type):
+ """Construct a `BinarySourceReference`."""
+ super(BinarySourceReference, self).__init__()
+ self.binary_package_release = binary_package_release
+ self.source_package_release = source_package_release
+ self.reference_type = reference_type
+
+
+@implementer(IBinarySourceReferenceSet)
+class BinarySourceReferenceSet:
+ """See `IBinarySourceReferenceSet`."""
+
+ @classmethod
+ def createFromRelationship(cls, bpr, relationship, reference_type):
+ """See `IBinarySourceReferenceSet`."""
+ if not relationship:
+ return []
+
+ try:
+ with warnings.catch_warnings():
+ warnings.simplefilter("error")
+ parsed_rel = PkgRelation.parse_relations(relationship)
+ except Warning as error:
+ raise UnparsableBuiltUsing(
+ "Invalid Built-Using field; cannot be parsed by deb822: %s"
+ % (error,))
+
+ build = bpr.build
+ dependencies = expand_dependencies(
+ build.archive, build.distro_arch_series, build.pocket,
+ build.current_component, bpr.sourcepackagename)
+
+ values = []
+ for or_rel in parsed_rel:
+ if len(or_rel) != 1:
+ raise UnparsableBuiltUsing(
+ "Alternatives are not allowed in Built-Using field: %s"
+ % (PkgRelation.str([or_rel]),))
+ rel = or_rel[0]
+ if rel["version"] is None or rel["version"][0] != "=":
+ raise UnparsableBuiltUsing(
+ "Built-Using must contain strict dependencies: %s"
+ % (PkgRelation.str([or_rel]),))
+ # "source-package-name (= version)" might refer to any of
+ # several SPRs, for example if the same source package was
+ # uploaded to a PPA and then uploaded separately (not copied -
+ # copies add new references to the same SPR) to the
+ # distribution's primary archive. We need to disambiguate this
+ # and find an actual SPR so that we can efficiently look up
+ # references for a given source publication. As an
+ # approximation, try this build's archive dependencies in order.
+ # This may go wrong, but rarely.
+ SPPH = SourcePackagePublishingHistory
+ SPN = SourcePackageName
+ SPR = SourcePackageRelease
+ dependencies_values = Values(
+ "dependencies",
+ [("index", "integer"),
+ ("archive", "integer"),
+ ("distroseries", "integer"),
+ ("pocket", "integer")],
+ [
+ (i, archive.id, das.distroseries.id,
+ dbify_value(SPPH.pocket, pocket))
+ for i, (archive, das, pocket, _) in enumerate(
+ dependencies)])
+ dependencies_table = Table("dependencies")
+ tables = [
+ SPPH,
+ Join(
+ dependencies_values,
+ And(
+ SPPH.archive == Column("archive", dependencies_table),
+ SPPH.distroseries ==
+ Column("distroseries", dependencies_table),
+ SPPH.pocket == Column("pocket", dependencies_table))),
+ Join(SPN, SPPH.sourcepackagename == SPN.id),
+ Join(SPR, SPPH.sourcepackagerelease == SPR.id),
+ ]
+ closest_spr_id = IStore(SPPH).using(*tables).find(
+ SPPH.sourcepackagereleaseID,
+ SPN.name == rel["name"],
+ SPR.version == rel["version"][1],
+ SPPH.status.is_in((
+ PackagePublishingStatus.PENDING,
+ PackagePublishingStatus.PUBLISHED,
+ PackagePublishingStatus.SUPERSEDED)),
+ ).order_by(Column("index", dependencies_table)).first()
+ if closest_spr_id is None:
+ raise UnparsableBuiltUsing(
+ "Built-Using refers to unknown or deleted source package "
+ "%s (= %s)" % (rel["name"], rel["version"][1]))
+ values.append((bpr.id, closest_spr_id, reference_type))
+
+ return create(
+ (BinarySourceReference.binary_package_release_id,
+ BinarySourceReference.source_package_release_id,
+ BinarySourceReference.reference_type),
+ values, get_objects=True)
+
+ @classmethod
+ def makeRelationship(cls, references):
+ """See `IBinarySourceReferenceSet`."""
+ return PkgRelation.str([
+ [{
+ "name": reference.source_package_release.name,
+ "version": ("=", reference.source_package_release.version),
+ }]
+ for reference in references])
+
+ @classmethod
+ def findByBinaryPackageRelease(cls, bpr, reference_type):
+ """See `IBinarySourceReferenceSet`."""
+ return IStore(BinarySourceReference).find(
+ BinarySourceReference,
+ BinarySourceReference.binary_package_release == bpr,
+ BinarySourceReference.reference_type == reference_type)
diff --git a/lib/lp/soyuz/scripts/gina/handlers.py b/lib/lp/soyuz/scripts/gina/handlers.py
index 66d2aeb..8cbc52b 100644
--- a/lib/lp/soyuz/scripts/gina/handlers.py
+++ b/lib/lp/soyuz/scripts/gina/handlers.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Gina db handlers.
@@ -52,10 +52,15 @@ from lp.services.librarian.interfaces import ILibraryFileAliasSet
from lp.services.scripts import log
from lp.soyuz.enums import (
BinaryPackageFormat,
+ BinarySourceReferenceType,
PackagePublishingStatus,
)
from lp.soyuz.interfaces.binarypackagebuild import IBinaryPackageBuildSet
from lp.soyuz.interfaces.binarypackagename import IBinaryPackageNameSet
+from lp.soyuz.interfaces.binarysourcereference import (
+ IBinarySourceReferenceSet,
+ UnparsableBuiltUsing,
+ )
from lp.soyuz.interfaces.publishing import (
active_publishing_status,
IPublishingSet,
@@ -820,6 +825,20 @@ class BinaryPackageHandler:
installedsize=bin.installed_size,
architecturespecific=architecturespecific,
**kwargs)
+ try:
+ getUtility(IBinarySourceReferenceSet).createFromRelationship(
+ binpkg, bin.built_using, BinarySourceReferenceType.BUILT_USING)
+ except UnparsableBuiltUsing:
+ # XXX cjwatson 2020-02-03: It might be nice if we created
+ # BinarySourceReference rows at least for those relations that
+ # can be parsed and resolved to SourcePackageReleases. It's not
+ # worth spending much time on given that we don't use binary
+ # imports much, though. For now, just stuff the whole field
+ # into user_defined_fields.
+ if binpkg._user_defined_fields is None:
+ binpkg._user_defined_fields = []
+ binpkg._user_defined_fields.append(
+ ("Built-Using", bin.built_using))
log.info('Binary Package Release %s (%s) created' %
(bin_name.name, bin.version))
diff --git a/lib/lp/soyuz/scripts/gina/packages.py b/lib/lp/soyuz/scripts/gina/packages.py
index 4126b80..332b564 100644
--- a/lib/lp/soyuz/scripts/gina/packages.py
+++ b/lib/lp/soyuz/scripts/gina/packages.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Package information classes.
@@ -466,6 +466,7 @@ class BinaryPackageData(AbstractPackageData):
pre_depends = ""
enhances = ""
breaks = ""
+ built_using = ""
essential = False
# Overwritten in do_package, optionally
diff --git a/lib/lp/soyuz/tests/test_binarysourcereference.py b/lib/lp/soyuz/tests/test_binarysourcereference.py
new file mode 100644
index 0000000..47c8f30
--- /dev/null
+++ b/lib/lp/soyuz/tests/test_binarysourcereference.py
@@ -0,0 +1,227 @@
+# Copyright 2020 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Test references from binary packages to source packages."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+
+import re
+
+from testtools.matchers import (
+ MatchesSetwise,
+ MatchesStructure,
+ )
+from testtools.testcase import ExpectedException
+from zope.component import getUtility
+
+from lp.registry.interfaces.pocket import PackagePublishingPocket
+from lp.soyuz.enums import (
+ ArchivePurpose,
+ BinarySourceReferenceType,
+ PackagePublishingStatus,
+ )
+from lp.soyuz.interfaces.binarysourcereference import (
+ IBinarySourceReferenceSet,
+ UnparsableBuiltUsing,
+ )
+from lp.testing import TestCaseWithFactory
+from lp.testing.layers import DatabaseFunctionalLayer
+
+
+class TestBinarySourceReference(TestCaseWithFactory):
+
+ layer = DatabaseFunctionalLayer
+
+ def setUp(self):
+ super(TestBinarySourceReference, self).setUp()
+ self.reference_set = getUtility(IBinarySourceReferenceSet)
+
+ def test_createFromRelationship_empty(self):
+ bpr = self.factory.makeBinaryPackageRelease()
+ self.assertEqual(
+ [],
+ self.reference_set.createFromRelationship(
+ bpr, "", BinarySourceReferenceType.BUILT_USING))
+
+ def test_createFromRelationship_nonsense(self):
+ bpr = self.factory.makeBinaryPackageRelease()
+ expected_message = (
+ r"Invalid Built-Using field; cannot be parsed by deb822: .*")
+ with ExpectedException(UnparsableBuiltUsing, expected_message):
+ self.reference_set.createFromRelationship(
+ bpr, "nonsense (", BinarySourceReferenceType.BUILT_USING)
+
+ def test_createFromRelationship_alternatives(self):
+ bpr = self.factory.makeBinaryPackageRelease()
+ expected_message = (
+ r"Alternatives are not allowed in Built-Using field: "
+ r"foo \(= 1\) \| bar \(= 2\)")
+ with ExpectedException(UnparsableBuiltUsing, expected_message):
+ self.reference_set.createFromRelationship(
+ bpr, "foo (= 1) | bar (= 2)",
+ BinarySourceReferenceType.BUILT_USING)
+
+ def test_createFromRelationship_no_version(self):
+ bpr = self.factory.makeBinaryPackageRelease()
+ expected_message = r"Built-Using must contain strict dependencies: foo"
+ with ExpectedException(UnparsableBuiltUsing, expected_message):
+ self.reference_set.createFromRelationship(
+ bpr, "foo", BinarySourceReferenceType.BUILT_USING)
+
+ def test_createFromRelationship_inequality(self):
+ bpr = self.factory.makeBinaryPackageRelease()
+ expected_message = (
+ r"Built-Using must contain strict dependencies: foo \(>= 1\)")
+ with ExpectedException(UnparsableBuiltUsing, expected_message):
+ self.reference_set.createFromRelationship(
+ bpr, "foo (>= 1)", BinarySourceReferenceType.BUILT_USING)
+
+ def test_createFromRelationship_unknown_source_package_name(self):
+ bpr = self.factory.makeBinaryPackageRelease()
+ relationship = "nonexistent (= 1)"
+ expected_message = (
+ r"Built-Using refers to unknown or deleted source package %s" %
+ re.escape(relationship))
+ with ExpectedException(UnparsableBuiltUsing, expected_message):
+ self.reference_set.createFromRelationship(
+ bpr, relationship, BinarySourceReferenceType.BUILT_USING)
+
+ def test_createFromRelationship_unknown_source_package_version(self):
+ bpr = self.factory.makeBinaryPackageRelease()
+ spph = self.factory.makeSourcePackagePublishingHistory(
+ archive=bpr.build.archive,
+ distroseries=bpr.build.distro_series,
+ component=bpr.build.current_component)
+ spr = spph.sourcepackagerelease
+ relationship = "%s (= %s.1)" % (spr.name, spr.version)
+ expected_message = (
+ r"Built-Using refers to unknown or deleted source package %s" %
+ re.escape(relationship))
+ with ExpectedException(UnparsableBuiltUsing, expected_message):
+ self.reference_set.createFromRelationship(
+ bpr, relationship, BinarySourceReferenceType.BUILT_USING)
+
+ def test_createFromRelationship_deleted(self):
+ bpr = self.factory.makeBinaryPackageRelease()
+ spph = self.factory.makeSourcePackagePublishingHistory(
+ archive=bpr.build.archive,
+ distroseries=bpr.build.distro_series,
+ component=bpr.build.current_component,
+ status=PackagePublishingStatus.DELETED)
+ spr = spph.sourcepackagerelease
+ relationship = "%s (= %s)" % (spr.name, spr.version)
+ expected_message = (
+ r"Built-Using refers to unknown or deleted source package %s" %
+ re.escape(relationship))
+ with ExpectedException(UnparsableBuiltUsing, expected_message):
+ self.reference_set.createFromRelationship(
+ bpr, relationship, BinarySourceReferenceType.BUILT_USING)
+
+ def test_createFromRelationship_simple(self):
+ bpr = self.factory.makeBinaryPackageRelease()
+ spphs = [
+ self.factory.makeSourcePackagePublishingHistory(
+ archive=bpr.build.archive,
+ distroseries=bpr.build.distro_series, pocket=bpr.build.pocket)
+ for _ in range(3)]
+ sprs = [spph.sourcepackagerelease for spph in spphs]
+ # Create a few more SPPHs with slight mismatches to ensure that
+ # createFromRelationship matches correctly.
+ self.factory.makeSourcePackagePublishingHistory(
+ archive=bpr.build.archive, pocket=bpr.build.pocket,
+ sourcepackagename=sprs[0].name, version=sprs[0].version)
+ self.factory.makeSourcePackagePublishingHistory(
+ archive=bpr.build.archive, distroseries=bpr.build.distro_series,
+ pocket=PackagePublishingPocket.BACKPORTS,
+ sourcepackagename=sprs[0].name, version=sprs[0].version)
+ self.factory.makeSourcePackagePublishingHistory(
+ archive=bpr.build.archive, distroseries=bpr.build.distro_series,
+ pocket=bpr.build.pocket, sourcepackagename=sprs[0].name)
+ self.factory.makeSourcePackagePublishingHistory(
+ archive=bpr.build.archive, distroseries=bpr.build.distro_series,
+ pocket=bpr.build.pocket, version=sprs[0].version)
+ self.factory.makeSourcePackagePublishingHistory()
+ relationship = (
+ "%s (= %s), %s (= %s)" %
+ (sprs[0].name, sprs[0].version, sprs[1].name, sprs[1].version))
+ bsrs = self.reference_set.createFromRelationship(
+ bpr, relationship, BinarySourceReferenceType.BUILT_USING)
+ self.assertThat(bsrs, MatchesSetwise(*(
+ MatchesStructure.byEquality(
+ binary_package_release=bpr,
+ source_package_release=spr,
+ reference_type=BinarySourceReferenceType.BUILT_USING)
+ for spr in sprs[:2])))
+
+ def test_createFromRelationship_multiple_archives(self):
+ # createFromRelationship prefers SPRs found earlier in the build's
+ # archive dependencies.
+ archive = self.factory.makeArchive(purpose=ArchivePurpose.PPA)
+ build = self.factory.makeBinaryPackageBuild(archive=archive)
+ bpr = self.factory.makeBinaryPackageRelease(build=build)
+ spph = self.factory.makeSourcePackagePublishingHistory(
+ archive=archive, distroseries=build.distro_series,
+ pocket=build.pocket)
+ spr = spph.sourcepackagerelease
+ self.factory.makeSourcePackagePublishingHistory(
+ archive=build.distro_series.main_archive,
+ distroseries=build.distro_series, pocket=build.pocket,
+ sourcepackagename=spr.name, version=spr.version)
+ relationship = "%s (= %s)" % (spr.name, spr.version)
+ bsrs = self.reference_set.createFromRelationship(
+ bpr, relationship, BinarySourceReferenceType.BUILT_USING)
+ self.assertThat(bsrs, MatchesSetwise(
+ MatchesStructure.byEquality(
+ binary_package_release=bpr,
+ source_package_release=spr,
+ reference_type=BinarySourceReferenceType.BUILT_USING)))
+
+ def test_findByBinaryPackageRelease_empty(self):
+ bpr = self.factory.makeBinaryPackageRelease()
+ self.assertContentEqual(
+ [],
+ self.reference_set.findByBinaryPackageRelease(
+ bpr, BinarySourceReferenceType.BUILT_USING))
+
+ def test_findByBinaryPackageRelease(self):
+ bprs = [self.factory.makeBinaryPackageRelease() for _ in range(2)]
+ all_sprs = []
+ for bpr in bprs:
+ spphs = [
+ self.factory.makeSourcePackagePublishingHistory(
+ archive=bpr.build.archive,
+ distroseries=bpr.build.distro_series,
+ pocket=bpr.build.pocket)
+ for _ in range(2)]
+ sprs = [spph.sourcepackagerelease for spph in spphs]
+ all_sprs.extend(sprs)
+ relationship = (
+ "%s (= %s), %s (= %s)" %
+ (sprs[0].name, sprs[0].version, sprs[1].name, sprs[1].version))
+ self.reference_set.createFromRelationship(
+ bpr, relationship, BinarySourceReferenceType.BUILT_USING)
+ other_bpr = self.factory.makeBinaryPackageRelease()
+ self.assertThat(
+ self.reference_set.findByBinaryPackageRelease(
+ bprs[0], BinarySourceReferenceType.BUILT_USING),
+ MatchesSetwise(*(
+ MatchesStructure.byEquality(
+ binary_package_release=bprs[0],
+ source_package_release=spr,
+ reference_type=BinarySourceReferenceType.BUILT_USING)
+ for spr in all_sprs[:2])))
+ self.assertThat(
+ self.reference_set.findByBinaryPackageRelease(
+ bprs[1], BinarySourceReferenceType.BUILT_USING),
+ MatchesSetwise(*(
+ MatchesStructure.byEquality(
+ binary_package_release=bprs[1],
+ source_package_release=spr,
+ reference_type=BinarySourceReferenceType.BUILT_USING)
+ for spr in all_sprs[2:])))
+ self.assertContentEqual(
+ [],
+ self.reference_set.findByBinaryPackageRelease(
+ other_bpr, BinarySourceReferenceType.BUILT_USING))
diff --git a/lib/lp/soyuz/tests/test_publishing.py b/lib/lp/soyuz/tests/test_publishing.py
index e65003d..af4ae5f 100644
--- a/lib/lp/soyuz/tests/test_publishing.py
+++ b/lib/lp/soyuz/tests/test_publishing.py
@@ -38,9 +38,13 @@ from lp.services.log.logger import DevNullLogger
from lp.soyuz.enums import (
ArchivePurpose,
BinaryPackageFormat,
+ BinarySourceReferenceType,
PackageUploadStatus,
)
from lp.soyuz.interfaces.binarypackagename import IBinaryPackageNameSet
+from lp.soyuz.interfaces.binarysourcereference import (
+ IBinarySourceReferenceSet,
+ )
from lp.soyuz.interfaces.component import IComponentSet
from lp.soyuz.interfaces.publishing import (
active_publishing_status,
@@ -306,7 +310,8 @@ class SoyuzTestPublisher:
shlibdep=None, depends=None, recommends=None,
suggests=None, conflicts=None, replaces=None,
provides=None, pre_depends=None, enhances=None,
- breaks=None, filecontent=b'bbbiiinnnaaarrryyy',
+ breaks=None, built_using=None,
+ filecontent=b'bbbiiinnnaaarrryyy',
changes_file_content=b"Fake: fake changes file",
status=PackagePublishingStatus.PENDING,
pocket=PackagePublishingPocket.RELEASE,
@@ -353,7 +358,8 @@ class SoyuzTestPublisher:
build, binaryname + '-dbgsym', filecontent, summary,
description, shlibdep, depends, recommends, suggests,
conflicts, replaces, provides, pre_depends, enhances,
- breaks, BinaryPackageFormat.DDEB, version=version)
+ breaks, built_using, BinaryPackageFormat.DDEB,
+ version=version)
pub_binaries += self.publishBinaryInArchive(
binarypackagerelease_ddeb, archive, status,
pocket, scheduleddeletiondate, dateremoved,
@@ -364,7 +370,7 @@ class SoyuzTestPublisher:
binarypackagerelease = self.uploadBinaryForBuild(
build, binaryname, filecontent, summary, description,
shlibdep, depends, recommends, suggests, conflicts, replaces,
- provides, pre_depends, enhances, breaks, format,
+ provides, pre_depends, enhances, breaks, built_using, format,
binarypackagerelease_ddeb, version=version,
user_defined_fields=user_defined_fields)
pub_binaries += self.publishBinaryInArchive(
@@ -387,8 +393,9 @@ class SoyuzTestPublisher:
summary="summary", description="description", shlibdep=None,
depends=None, recommends=None, suggests=None, conflicts=None,
replaces=None, provides=None, pre_depends=None, enhances=None,
- breaks=None, format=BinaryPackageFormat.DEB, debug_package=None,
- user_defined_fields=None, homepage=None, version=None):
+ breaks=None, built_using=None, format=BinaryPackageFormat.DEB,
+ debug_package=None, user_defined_fields=None, homepage=None,
+ version=None):
"""Return the corresponding `BinaryPackageRelease`."""
sourcepackagerelease = build.source_package_release
distroarchseries = build.distro_arch_series
@@ -418,6 +425,7 @@ class SoyuzTestPublisher:
pre_depends=pre_depends,
enhances=enhances,
breaks=breaks,
+ built_using=built_using,
essential=False,
installedsize=100,
architecturespecific=architecturespecific,
Follow ups
