launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #28753
[Merge] ~cjwatson/launchpad:ci-build-upload-make-spr into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:ci-build-upload-make-spr into launchpad:master with ~cjwatson/launchpad:refactor-ci-build-upload-job-scan-file as a prerequisite.
Commit message:
Create a source publication when uploading a CI build
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/426502
This has some advantages and disadvantages, but on the whole I think the advantages outweigh the disadvantages.
The downsides are:
* if the CI build produces nothing that looks like source code at all, then we end up with a slightly odd "source package" with no files attached to it;
* if builds do produce something that looks like source code (e.g. a Python sdist), then they're on the honour system to either produce it from only one architecture or to produce something equivalent from all architectures.
On the other hand:
* Launchpad's data model for packages makes the assumption in many places that binary packages have an associated source package, and my attempts to make it tolerate isolated binary packages have only been marginally successful;
* treating source-like artifacts (Python sdists, zip files of Go source code, etc.) as source packages rather than binary packages makes more sense to humans;
* source packages provide a useful grouping for collections of binary packages from builds of the same version;
* creating source publications means that the PPA web UI works for PPAs populated from CI builds with only a few small adjustments (not in this branch);
* creating source publications means that it should be straightforward to get the existing package copying mechanism to work.
I've slightly cheekily renamed `SourcePackageType.SDIST` to `SourcePackageType.CI_BUILD`. The type of an individual source file is represented separately using `SourcePackageFileType`, and since the data model's assumptions generally involve a single source with zero or more binaries, it's simpler for `SourcePackageRelease.format` not to need to change depending on the types of the individual files it contains. There was previously no way to create a source package with `format=SDIST` outside the test suite, and a query on staging returns zero rows.
As a concrete example of a source package file type and in order to allow more useful testing, I've added support for uploading Python sdists from CI builds as part of this branch.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:ci-build-upload-make-spr into launchpad:master.
diff --git a/lib/lp/_schema_circular_imports.py b/lib/lp/_schema_circular_imports.py
index 4c17b6b..4cf6e63 100644
--- a/lib/lp/_schema_circular_imports.py
+++ b/lib/lp/_schema_circular_imports.py
@@ -488,6 +488,7 @@ patch_reference_property(IPackageUpload, 'copy_source_archive', IArchive)
patch_reference_property(
ISourcePackageRelease, 'source_package_recipe_build',
ISourcePackageRecipeBuild)
+patch_reference_property(ISourcePackageRelease, 'ci_build', ICIBuild)
# ISourcePackageRecipeView
patch_entry_return_type(
diff --git a/lib/lp/archivepublisher/tests/test_artifactory.py b/lib/lp/archivepublisher/tests/test_artifactory.py
index c9feb58..18af683 100644
--- a/lib/lp/archivepublisher/tests/test_artifactory.py
+++ b/lib/lp/archivepublisher/tests/test_artifactory.py
@@ -576,7 +576,7 @@ class TestArtifactoryPoolFromLibrarian(TestCaseWithFactory):
sourcepackagename="foo",
version="1.0",
channel="edge",
- format=SourcePackageType.SDIST,
+ format=SourcePackageType.CI_BUILD,
)
spr = spph.sourcepackagerelease
sprf = self.factory.makeSourcePackageReleaseFile(
@@ -635,7 +635,7 @@ class TestArtifactoryPoolFromLibrarian(TestCaseWithFactory):
archive=pool.archive,
sourcepackagename="foo",
version="1.0",
- format=SourcePackageType.SDIST,
+ format=SourcePackageType.CI_BUILD,
)
bpph = self.factory.makeBinaryPackagePublishingHistory(
archive=pool.archive,
@@ -940,7 +940,7 @@ class TestArtifactoryPoolFromLibrarian(TestCaseWithFactory):
archive=pool.archive,
sourcepackagename="foo",
version="1.0",
- format=SourcePackageType.SDIST,
+ format=SourcePackageType.CI_BUILD,
)
bpph = self.factory.makeBinaryPackagePublishingHistory(
archive=pool.archive,
diff --git a/lib/lp/code/interfaces/cibuild.py b/lib/lp/code/interfaces/cibuild.py
index 69c765b..d929283 100644
--- a/lib/lp/code/interfaces/cibuild.py
+++ b/lib/lp/code/interfaces/cibuild.py
@@ -133,6 +133,10 @@ class ICIBuildView(IPackageBuildView, IPrivacy):
"A mapping from job IDs to result tokens, retrieved from the "
"builder.")))
+ sourcepackages = Attribute(
+ "A list of source packages that resulted from this build, ordered by "
+ "name.")
+
binarypackages = Attribute(
"A list of binary packages that resulted from this build, ordered by "
"name.")
@@ -182,6 +186,14 @@ class ICIBuildView(IPackageBuildView, IPrivacy):
:return: A collection of URLs for this build.
"""
+ def createSourcePackageRelease(
+ distroseries, sourcepackagename, version, creator=None,
+ archive=None):
+ """Create and return a `SourcePackageRelease` for this CI build.
+
+ The new source package release will be linked to this build.
+ """
+
def createBinaryPackageRelease(
binarypackagename, version, summary, description, binpackageformat,
architecturespecific, installedsize=None, homepage=None,
diff --git a/lib/lp/code/model/cibuild.py b/lib/lp/code/model/cibuild.py
index 76ccfc3..281c3a1 100644
--- a/lib/lp/code/model/cibuild.py
+++ b/lib/lp/code/model/cibuild.py
@@ -65,8 +65,10 @@ from lp.code.model.gitref import GitRef
from lp.code.model.lpcraft import load_configuration
from lp.registry.interfaces.pocket import PackagePublishingPocket
from lp.registry.interfaces.series import SeriesStatus
+from lp.registry.interfaces.sourcepackage import SourcePackageType
from lp.registry.model.distribution import Distribution
from lp.registry.model.distroseries import DistroSeries
+from lp.registry.model.sourcepackagename import SourcePackageName
from lp.services.database.bulk import load_related
from lp.services.database.constants import DEFAULT
from lp.services.database.decoratedresultset import DecoratedResultSet
@@ -91,6 +93,7 @@ from lp.services.propertycache import cachedproperty
from lp.soyuz.model.binarypackagename import BinaryPackageName
from lp.soyuz.model.binarypackagerelease import BinaryPackageRelease
from lp.soyuz.model.distroarchseries import DistroArchSeries
+from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
def get_stages(configuration):
@@ -471,6 +474,17 @@ class CIBuild(PackageBuildMixin, StormBase):
# We don't currently send any notifications.
@property
+ def sourcepackages(self):
+ """See `ICIBuild`."""
+ releases = IStore(SourcePackageRelease).find(
+ (SourcePackageRelease, SourcePackageName),
+ SourcePackageRelease.ci_build == self,
+ SourcePackageRelease.sourcepackagename == SourcePackageName.id)
+ releases = releases.order_by(
+ SourcePackageName.name, SourcePackageRelease.id)
+ return DecoratedResultSet(releases, result_decorator=itemgetter(0))
+
+ @property
def binarypackages(self):
"""See `ICIBuild`."""
releases = IStore(BinaryPackageRelease).find(
@@ -481,6 +495,22 @@ class CIBuild(PackageBuildMixin, StormBase):
BinaryPackageName.name, BinaryPackageRelease.id)
return DecoratedResultSet(releases, result_decorator=itemgetter(0))
+ def createSourcePackageRelease(
+ self, distroseries, sourcepackagename, version, creator=None,
+ archive=None):
+ """See `ICIBuild`."""
+ return distroseries.createUploadedSourcePackageRelease(
+ sourcepackagename=sourcepackagename,
+ version=version,
+ format=SourcePackageType.CI_BUILD,
+ # This doesn't really make sense for SPRs created for CI builds,
+ # but the column is NOT NULL. The empty string will do though,
+ # since nothing will use this.
+ architecturehintlist="",
+ creator=creator,
+ archive=archive,
+ ci_build=self)
+
def createBinaryPackageRelease(
self, binarypackagename, version, summary, description,
binpackageformat, architecturespecific, installedsize=None,
diff --git a/lib/lp/code/model/tests/test_cibuild.py b/lib/lp/code/model/tests/test_cibuild.py
index 89a432c..ca0192b 100644
--- a/lib/lp/code/model/tests/test_cibuild.py
+++ b/lib/lp/code/model/tests/test_cibuild.py
@@ -62,6 +62,7 @@ from lp.code.model.cibuild import (
from lp.code.model.lpcraft import load_configuration
from lp.code.tests.helpers import GitHostingFixture
from lp.registry.interfaces.series import SeriesStatus
+from lp.registry.interfaces.sourcepackage import SourcePackageType
from lp.services.authserver.xmlrpc import AuthServerAPIView
from lp.services.config import config
from lp.services.librarian.browser import ProxiedLibraryFileAlias
@@ -400,6 +401,27 @@ class TestCIBuild(TestCaseWithFactory):
commit_sha1=build.commit_sha1,
ci_build=build))
+ def test_createSourcePackageRelease(self):
+ distroseries = self.factory.makeDistroSeries()
+ archive = self.factory.makeArchive(
+ distribution=distroseries.distribution)
+ build = self.factory.makeCIBuild()
+ spn = self.factory.makeSourcePackageName()
+ spr = build.createSourcePackageRelease(
+ distroseries, spn, "1.0", creator=build.git_repository.owner,
+ archive=archive)
+ self.assertThat(spr, MatchesStructure(
+ upload_distroseries=Equals(distroseries),
+ sourcepackagename=Equals(spn),
+ version=Equals("1.0"),
+ format=Equals(SourcePackageType.CI_BUILD),
+ architecturehintlist=Equals(""),
+ creator=Equals(build.git_repository.owner),
+ upload_archive=Equals(archive),
+ ci_build=Equals(build),
+ ))
+ self.assertContentEqual([spr], build.sourcepackages)
+
def test_createBinaryPackageRelease(self):
build = self.factory.makeCIBuild()
bpn = self.factory.makeBinaryPackageName()
diff --git a/lib/lp/registry/interfaces/sourcepackage.py b/lib/lp/registry/interfaces/sourcepackage.py
index fcde488..6a7e707 100644
--- a/lib/lp/registry/interfaces/sourcepackage.py
+++ b/lib/lp/registry/interfaces/sourcepackage.py
@@ -456,10 +456,10 @@ class SourcePackageType(DBEnumeratedType):
This is the source package format used by Gentoo.
""")
- SDIST = DBItem(4, """
- The Python Format
+ CI_BUILD = DBItem(4, """
+ CI Build
- This is the source package format used by Python packages.
+ An ad-hoc source package generated by a CI build.
""")
diff --git a/lib/lp/soyuz/interfaces/sourcepackagerelease.py b/lib/lp/soyuz/interfaces/sourcepackagerelease.py
index 8e6cee9..aac4f39 100644
--- a/lib/lp/soyuz/interfaces/sourcepackagerelease.py
+++ b/lib/lp/soyuz/interfaces/sourcepackagerelease.py
@@ -145,11 +145,20 @@ class ISourcePackageRelease(Interface):
# Really ISourcePackageRecipeBuild -- see _schema_circular_imports.
source_package_recipe_build = Reference(
schema=Interface,
- description=_("The `SourcePackageRecipeBuild` which produced this "
- "source package release, or None if it was created from a "
- "traditional upload."),
+ description=_(
+ "The `SourcePackageRecipeBuild` which produced this source "
+ "package release, or None if it was not created from a source "
+ "package recipe."),
title=_("Source package recipe build"),
required=False, readonly=True)
+ # Really ICIBuild, patched in _schema_circular_imports.
+ ci_build = Reference(
+ schema=Interface,
+ description=_(
+ "The `CIBuild` which produced this source package release, or "
+ "None if it was not created from a CI build."),
+ title=_("CI build"),
+ required=False, readonly=True)
def getUserDefinedField(name):
"""Case-insensitively get a user-defined field."""
diff --git a/lib/lp/soyuz/model/archivejob.py b/lib/lp/soyuz/model/archivejob.py
index 65a62af..6fb7c1f 100644
--- a/lib/lp/soyuz/model/archivejob.py
+++ b/lib/lp/soyuz/model/archivejob.py
@@ -11,7 +11,10 @@ import tempfile
import zipfile
from lazr.delegates import delegate_to
-from pkginfo import Wheel
+from pkginfo import (
+ SDist,
+ Wheel,
+ )
from storm.expr import And
from storm.locals import (
Int,
@@ -34,6 +37,7 @@ from lp.registry.interfaces.distributionsourcepackage import (
)
from lp.registry.interfaces.distroseries import IDistroSeriesSet
from lp.registry.interfaces.pocket import PackagePublishingPocket
+from lp.registry.interfaces.sourcepackage import SourcePackageFileType
from lp.services.config import config
from lp.services.database.enumcol import DBEnum
from lp.services.database.interfaces import IMasterStore
@@ -205,9 +209,14 @@ class ScanException(Exception):
class ScannedArtifact:
- def __init__(self, *, artifact, metadata):
+ def __init__(self, *, artifact, metadata, is_binary):
self.artifact = artifact
self.metadata = metadata
+ self.is_binary = is_binary
+
+ @property
+ def version(self):
+ return self.metadata["version"]
@implementer(ICIBuildUploadJob)
@@ -236,13 +245,16 @@ class CIBuildUploadJob(ArchiveJobDerived):
# We're only interested in uploading certain kinds of packages to
# certain kinds of archives.
- binary_format_by_repository_format = {
+ format_by_repository_format = {
ArchiveRepositoryFormat.DEBIAN: {
BinaryPackageFormat.DEB,
BinaryPackageFormat.UDEB,
BinaryPackageFormat.DDEB,
},
- ArchiveRepositoryFormat.PYTHON: {BinaryPackageFormat.WHL},
+ ArchiveRepositoryFormat.PYTHON: {
+ SourcePackageFileType.SDIST,
+ BinaryPackageFormat.WHL,
+ },
ArchiveRepositoryFormat.CONDA: {
BinaryPackageFormat.CONDA_V1,
BinaryPackageFormat.CONDA_V2,
@@ -325,17 +337,34 @@ class CIBuildUploadJob(ArchiveJobDerived):
os.path.basename(path), e)
return None
return {
+ "is_binary": True,
+ "format": BinaryPackageFormat.WHL,
"name": wheel.name,
"version": wheel.version,
"summary": wheel.summary or "",
"description": wheel.description,
- "binpackageformat": BinaryPackageFormat.WHL,
"architecturespecific": "any" not in parsed_path.platform_tags,
"homepage": wheel.home_page or "",
}
+ def _scanSDist(self, path):
+ try:
+ sdist = SDist(path)
+ except Exception as e:
+ logger.warning(
+ "Failed to scan %s as a Python sdist: %s",
+ os.path.basename(path), e)
+ return None
+ return {
+ "is_binary": False,
+ "format": SourcePackageFileType.SDIST,
+ "name": sdist.name,
+ "version": sdist.version,
+ }
+
def _scanCondaMetadata(self, index, about):
return {
+ "is_binary": True,
"name": index["name"],
"version": index["version"],
"summary": about.get("summary", ""),
@@ -361,7 +390,7 @@ class CIBuildUploadJob(ArchiveJobDerived):
"Failed to scan %s as a Conda v1 package: %s",
os.path.basename(path), e)
return None
- scanned = {"binpackageformat": BinaryPackageFormat.CONDA_V1}
+ scanned = {"format": BinaryPackageFormat.CONDA_V1}
scanned.update(self._scanCondaMetadata(index, about))
return scanned
@@ -383,13 +412,15 @@ class CIBuildUploadJob(ArchiveJobDerived):
"Failed to scan %s as a Conda v2 package: %s",
os.path.basename(path), e)
return None
- scanned = {"binpackageformat": BinaryPackageFormat.CONDA_V2}
+ scanned = {"format": BinaryPackageFormat.CONDA_V2}
scanned.update(self._scanCondaMetadata(index, about))
return scanned
def _scanFile(self, path):
_scanners = (
(".whl", self._scanWheel),
+ (".tar.gz", self._scanSDist),
+ (".zip", self._scanSDist),
(".tar.bz2", self._scanCondaV1),
(".conda", self._scanCondaV2),
)
@@ -414,8 +445,8 @@ class CIBuildUploadJob(ArchiveJobDerived):
Returns a list of `ScannedArtifact`s containing metadata for
relevant artifacts.
"""
- allowed_binary_formats = (
- self.binary_format_by_repository_format.get(
+ allowed_formats = (
+ self.format_by_repository_format.get(
self.archive.repository_format, set()))
scanned = []
with tempfile.TemporaryDirectory(prefix="ci-build-copy-job") as tmpdir:
@@ -429,30 +460,80 @@ class CIBuildUploadJob(ArchiveJobDerived):
metadata = self._scanFile(contents)
if metadata is None:
continue
- if metadata["binpackageformat"] not in allowed_binary_formats:
+ if metadata["format"] not in allowed_formats:
logger.info(
"Skipping %s (not relevant to %s archives)",
name, self.archive.repository_format)
continue
+ is_binary = metadata.pop("is_binary")
scanned.append(
- ScannedArtifact(artifact=artifact, metadata=metadata))
+ ScannedArtifact(
+ artifact=artifact, metadata=metadata,
+ is_binary=is_binary))
return scanned
+ def _uploadSources(self, scanned):
+ """Upload sources from an iterable of `ScannedArtifact`s."""
+ # Launchpad's data model generally assumes that a single source is
+ # associated with multiple binaries. However, a source package
+ # release can have multiple (or indeed no) files attached to it, so
+ # we make use of that if necessary.
+ releases = {
+ release.sourcepackagename: release
+ for release in self.ci_build.sourcepackages}
+ distroseries = self.ci_build.distro_arch_series.distroseries
+ build_target = self.ci_build.git_repository.target
+ spr = releases.get(build_target.sourcepackagename)
+ if spr is None:
+ spr = self.ci_build.createSourcePackageRelease(
+ distroseries=distroseries,
+ sourcepackagename=build_target.sourcepackagename,
+ # We don't have a good concept of source version here, but
+ # the data model demands one. Arbitrarily pick the version
+ # of the first scanned artifact.
+ version=scanned[0].version,
+ creator=self.requester,
+ archive=self.archive)
+ for scanned_artifact in scanned:
+ if scanned_artifact.is_binary:
+ continue
+ library_file = scanned_artifact.artifact.library_file
+ logger.info(
+ "Uploading %s to %s %s (%s)",
+ library_file.filename, self.archive.reference,
+ self.target_distroseries.getSuite(self.target_pocket),
+ self.target_channel)
+ filetype = scanned_artifact.metadata["format"]
+ for sprf in spr.files:
+ if (sprf.libraryfile == library_file and
+ sprf.filetype == filetype):
+ break
+ else:
+ spr.addFile(library_file, filetype=filetype)
+ getUtility(IPublishingSet).newSourcePublication(
+ archive=self.archive, sourcepackagerelease=spr,
+ distroseries=self.target_distroseries, pocket=self.target_pocket,
+ creator=self.requester, channel=self.target_channel)
+
def _uploadBinaries(self, scanned):
- """Upload an iterable of `ScannedArtifact`s to an archive."""
+ """Upload binaries from an iterable of `ScannedArtifact`s."""
releases = {
(release.binarypackagename, release.binpackageformat): release
for release in self.ci_build.binarypackages}
binaries = OrderedDict()
for scanned_artifact in scanned:
+ if not scanned_artifact.is_binary:
+ continue
library_file = scanned_artifact.artifact.library_file
metadata = dict(scanned_artifact.metadata)
- binpackageformat = metadata["binpackageformat"]
+ binpackageformat = metadata["format"]
logger.info(
"Uploading %s to %s %s (%s)",
library_file.filename, self.archive.reference,
self.target_distroseries.getSuite(self.target_pocket),
self.target_channel)
+ metadata["binpackageformat"] = binpackageformat
+ del metadata["format"]
metadata["binarypackagename"] = bpn = (
getUtility(IBinaryPackageNameSet).ensure(metadata["name"]))
del metadata["name"]
@@ -488,6 +569,7 @@ class CIBuildUploadJob(ArchiveJobDerived):
self.ci_build)
scanned = self._scanArtifacts(artifacts)
if scanned:
+ self._uploadSources(scanned)
self._uploadBinaries(scanned)
else:
names = [artifact.library_file.filename for artifact in artifacts]
diff --git a/lib/lp/soyuz/tests/test_archivejob.py b/lib/lp/soyuz/tests/test_archivejob.py
index b744bc8..b1c7acf 100644
--- a/lib/lp/soyuz/tests/test_archivejob.py
+++ b/lib/lp/soyuz/tests/test_archivejob.py
@@ -7,6 +7,7 @@ from debian.deb822 import Changes
from fixtures import (
FakeLogger,
MockPatch,
+ MockPatchObject,
)
from testtools.matchers import (
ContainsDict,
@@ -20,6 +21,10 @@ import transaction
from lp.code.enums import RevisionStatusArtifactType
from lp.code.model.revisionstatus import RevisionStatusArtifact
from lp.registry.interfaces.pocket import PackagePublishingPocket
+from lp.registry.interfaces.sourcepackage import (
+ SourcePackageFileType,
+ SourcePackageType,
+ )
from lp.services.config import config
from lp.services.database.interfaces import IStore
from lp.services.features.testing import FeatureFixture
@@ -243,11 +248,12 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
PackagePublishingPocket.RELEASE, target_channel="edge")
path = "wheel-indep/dist/wheel_indep-0.0.1-py3-none-any.whl"
expected = {
+ "is_binary": True,
+ "format": BinaryPackageFormat.WHL,
"name": "wheel-indep",
"version": "0.0.1",
"summary": "Example description",
"description": "Example long description\n",
- "binpackageformat": BinaryPackageFormat.WHL,
"architecturespecific": False,
"homepage": "",
}
@@ -263,16 +269,34 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
PackagePublishingPocket.RELEASE, target_channel="edge")
path = "wheel-arch/dist/wheel_arch-0.0.1-cp310-cp310-linux_x86_64.whl"
expected = {
+ "is_binary": True,
+ "format": BinaryPackageFormat.WHL,
"name": "wheel-arch",
"version": "0.0.1",
"summary": "Example description",
"description": "Example long description\n",
- "binpackageformat": BinaryPackageFormat.WHL,
"architecturespecific": True,
"homepage": "http://example.com/";,
}
self.assertEqual(expected, job._scanFile(datadir(path)))
+ def test__scanFile_sdist(self):
+ archive = self.factory.makeArchive()
+ distroseries = self.factory.makeDistroSeries(
+ distribution=archive.distribution)
+ build = self.makeCIBuild(archive.distribution)
+ job = CIBuildUploadJob.create(
+ build, build.git_repository.owner, archive, distroseries,
+ PackagePublishingPocket.RELEASE, target_channel="edge")
+ path = "wheel-arch/dist/wheel-arch-0.0.1.tar.gz"
+ expected = {
+ "is_binary": False,
+ "format": SourcePackageFileType.SDIST,
+ "name": "wheel-arch",
+ "version": "0.0.1",
+ }
+ self.assertEqual(expected, job._scanFile(datadir(path)))
+
def test__scanFile_conda_indep(self):
archive = self.factory.makeArchive()
distroseries = self.factory.makeDistroSeries(
@@ -283,11 +307,12 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
PackagePublishingPocket.RELEASE, target_channel="edge")
path = "conda-indep/dist/noarch/conda-indep-0.1-0.tar.bz2"
expected = {
+ "is_binary": True,
+ "format": BinaryPackageFormat.CONDA_V1,
"name": "conda-indep",
"version": "0.1",
"summary": "Example summary",
"description": "Example description",
- "binpackageformat": BinaryPackageFormat.CONDA_V1,
"architecturespecific": False,
"homepage": "",
"user_defined_fields": [("subdir", "noarch")],
@@ -304,11 +329,12 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
PackagePublishingPocket.RELEASE, target_channel="edge")
path = "conda-arch/dist/linux-64/conda-arch-0.1-0.tar.bz2"
expected = {
+ "is_binary": True,
+ "format": BinaryPackageFormat.CONDA_V1,
"name": "conda-arch",
"version": "0.1",
"summary": "Example summary",
"description": "Example description",
- "binpackageformat": BinaryPackageFormat.CONDA_V1,
"architecturespecific": True,
"homepage": "http://example.com/";,
"user_defined_fields": [("subdir", "linux-64")],
@@ -325,11 +351,12 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
PackagePublishingPocket.RELEASE, target_channel="edge")
path = "conda-v2-indep/dist/noarch/conda-v2-indep-0.1-0.conda"
expected = {
+ "is_binary": True,
+ "format": BinaryPackageFormat.CONDA_V2,
"name": "conda-v2-indep",
"version": "0.1",
"summary": "Example summary",
"description": "Example description",
- "binpackageformat": BinaryPackageFormat.CONDA_V2,
"architecturespecific": False,
"homepage": "",
"user_defined_fields": [("subdir", "noarch")],
@@ -346,11 +373,12 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
PackagePublishingPocket.RELEASE, target_channel="edge")
path = "conda-v2-arch/dist/linux-64/conda-v2-arch-0.1-0.conda"
expected = {
+ "is_binary": True,
+ "format": BinaryPackageFormat.CONDA_V2,
"name": "conda-v2-arch",
"version": "0.1",
"summary": "Example summary",
"description": "Example description",
- "binpackageformat": BinaryPackageFormat.CONDA_V2,
"architecturespecific": True,
"homepage": "http://example.com/";,
"user_defined_fields": [("subdir", "linux-64")],
@@ -384,6 +412,21 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
with dbuser(job.config.dbuser):
JobRunner([job]).runAll()
+ self.assertThat(archive.getPublishedSources(), MatchesSetwise(
+ MatchesStructure(
+ sourcepackagename=MatchesStructure.byEquality(
+ name=build.git_repository.target.name),
+ sourcepackagerelease=MatchesStructure(
+ ci_build=Equals(build),
+ sourcepackagename=MatchesStructure.byEquality(
+ name=build.git_repository.target.name),
+ version=Equals("0.0.1"),
+ format=Equals(SourcePackageType.CI_BUILD),
+ architecturehintlist=Equals(""),
+ creator=Equals(build.git_repository.owner),
+ files=Equals([])),
+ format=Equals(SourcePackageType.CI_BUILD),
+ distroseries=Equals(distroseries))))
self.assertThat(archive.getAllPublishedBinaries(), MatchesSetwise(*(
MatchesStructure(
binarypackagename=MatchesStructure.byEquality(
@@ -418,13 +461,17 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
archive.distribution, distro_arch_series=dases[0])
report = build.getOrCreateRevisionStatusReport("build:0")
report.setLog(b"log data")
- path = "wheel-arch/dist/wheel_arch-0.0.1-cp310-cp310-linux_x86_64.whl"
- with open(datadir(path), mode="rb") as f:
- report.attach(name=os.path.basename(path), data=f.read())
- artifact = IStore(RevisionStatusArtifact).find(
+ sdist_path = "wheel-arch/dist/wheel-arch-0.0.1.tar.gz"
+ wheel_path = (
+ "wheel-arch/dist/wheel_arch-0.0.1-cp310-cp310-linux_x86_64.whl")
+ with open(datadir(sdist_path), mode="rb") as f:
+ report.attach(name=os.path.basename(sdist_path), data=f.read())
+ with open(datadir(wheel_path), mode="rb") as f:
+ report.attach(name=os.path.basename(wheel_path), data=f.read())
+ artifacts = IStore(RevisionStatusArtifact).find(
RevisionStatusArtifact,
report=report,
- artifact_type=RevisionStatusArtifactType.BINARY).one()
+ artifact_type=RevisionStatusArtifactType.BINARY).order_by("id")
job = CIBuildUploadJob.create(
build, build.git_repository.owner, archive, distroseries,
PackagePublishingPocket.RELEASE, target_channel="edge")
@@ -433,6 +480,24 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
with dbuser(job.config.dbuser):
JobRunner([job]).runAll()
+ self.assertThat(archive.getPublishedSources(), MatchesSetwise(
+ MatchesStructure(
+ sourcepackagename=MatchesStructure.byEquality(
+ name=build.git_repository.target.name),
+ sourcepackagerelease=MatchesStructure(
+ ci_build=Equals(build),
+ sourcepackagename=MatchesStructure.byEquality(
+ name=build.git_repository.target.name),
+ version=Equals("0.0.1"),
+ format=Equals(SourcePackageType.CI_BUILD),
+ architecturehintlist=Equals(""),
+ creator=Equals(build.git_repository.owner),
+ files=MatchesSetwise(
+ MatchesStructure.byEquality(
+ libraryfile=artifacts[0].library_file,
+ filetype=SourcePackageFileType.SDIST))),
+ format=Equals(SourcePackageType.CI_BUILD),
+ distroseries=Equals(distroseries))))
self.assertThat(archive.getAllPublishedBinaries(), MatchesSetwise(
MatchesStructure(
binarypackagename=MatchesStructure.byEquality(
@@ -449,7 +514,7 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
homepage=Equals("http://example.com/";),
files=MatchesSetwise(
MatchesStructure.byEquality(
- libraryfile=artifact.library_file,
+ libraryfile=artifacts[1].library_file,
filetype=BinaryPackageFileType.WHL))),
binarypackageformat=Equals(BinaryPackageFormat.WHL),
distroarchseries=Equals(dases[0]))))
@@ -481,6 +546,21 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
with dbuser(job.config.dbuser):
JobRunner([job]).runAll()
+ self.assertThat(archive.getPublishedSources(), MatchesSetwise(
+ MatchesStructure(
+ sourcepackagename=MatchesStructure.byEquality(
+ name=build.git_repository.target.name),
+ sourcepackagerelease=MatchesStructure(
+ ci_build=Equals(build),
+ sourcepackagename=MatchesStructure.byEquality(
+ name=build.git_repository.target.name),
+ version=Equals("0.1"),
+ format=Equals(SourcePackageType.CI_BUILD),
+ architecturehintlist=Equals(""),
+ creator=Equals(build.git_repository.owner),
+ files=Equals([])),
+ format=Equals(SourcePackageType.CI_BUILD),
+ distroseries=Equals(distroseries))))
self.assertThat(archive.getAllPublishedBinaries(), MatchesSetwise(
MatchesStructure(
binarypackagename=MatchesStructure.byEquality(
@@ -529,6 +609,21 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
with dbuser(job.config.dbuser):
JobRunner([job]).runAll()
+ self.assertThat(archive.getPublishedSources(), MatchesSetwise(
+ MatchesStructure(
+ sourcepackagename=MatchesStructure.byEquality(
+ name=build.git_repository.target.name),
+ sourcepackagerelease=MatchesStructure(
+ ci_build=Equals(build),
+ sourcepackagename=MatchesStructure.byEquality(
+ name=build.git_repository.target.name),
+ version=Equals("0.1"),
+ format=Equals(SourcePackageType.CI_BUILD),
+ architecturehintlist=Equals(""),
+ creator=Equals(build.git_repository.owner),
+ files=Equals([])),
+ format=Equals(SourcePackageType.CI_BUILD),
+ distroseries=Equals(distroseries))))
self.assertThat(archive.getAllPublishedBinaries(), MatchesSetwise(
MatchesStructure(
binarypackagename=MatchesStructure.byEquality(
@@ -550,7 +645,7 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
binarypackageformat=Equals(BinaryPackageFormat.CONDA_V2),
distroarchseries=Equals(dases[0]))))
- def test_existing_release(self):
+ def test_existing_source_and_binary_releases(self):
# A `CIBuildUploadJob` can be run even if the build in question was
# already uploaded somewhere, and in that case may add publications
# in other locations for the same package.
@@ -561,13 +656,16 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
das = self.factory.makeDistroArchSeries(distroseries=distroseries)
build = self.makeCIBuild(archive.distribution, distro_arch_series=das)
report = build.getOrCreateRevisionStatusReport("build:0")
- path = "wheel-indep/dist/wheel_indep-0.0.1-py3-none-any.whl"
- with open(datadir(path), mode="rb") as f:
- report.attach(name=os.path.basename(path), data=f.read())
- artifact = IStore(RevisionStatusArtifact).find(
+ sdist_path = "wheel-indep/dist/wheel-indep-0.0.1.tar.gz"
+ with open(datadir(sdist_path), mode="rb") as f:
+ report.attach(name=os.path.basename(sdist_path), data=f.read())
+ wheel_path = "wheel-indep/dist/wheel_indep-0.0.1-py3-none-any.whl"
+ with open(datadir(wheel_path), mode="rb") as f:
+ report.attach(name=os.path.basename(wheel_path), data=f.read())
+ artifacts = IStore(RevisionStatusArtifact).find(
RevisionStatusArtifact,
report=report,
- artifact_type=RevisionStatusArtifactType.BINARY).one()
+ artifact_type=RevisionStatusArtifactType.BINARY).order_by("id")
job = CIBuildUploadJob.create(
build, build.git_repository.owner, archive, distroseries,
PackagePublishingPocket.RELEASE, target_channel="edge")
@@ -582,12 +680,103 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
with dbuser(job.config.dbuser):
JobRunner([job]).runAll()
+ spphs = archive.getPublishedSources()
+ # The source publications are for the same source package release,
+ # which has a single file attached to it.
+ self.assertEqual(1, len({spph.sourcepackagename for spph in spphs}))
+ self.assertEqual(1, len({spph.sourcepackagerelease for spph in spphs}))
+ self.assertThat(spphs, MatchesSetwise(*(
+ MatchesStructure(
+ sourcepackagename=MatchesStructure.byEquality(
+ name=build.git_repository.target.name),
+ sourcepackagerelease=MatchesStructure(
+ ci_build=Equals(build),
+ files=MatchesSetwise(
+ MatchesStructure.byEquality(
+ libraryfile=artifacts[0].library_file,
+ filetype=SourcePackageFileType.SDIST))),
+ format=Equals(SourcePackageType.CI_BUILD),
+ distroseries=Equals(distroseries),
+ channel=Equals(channel))
+ for channel in ("edge", "0.0.1/edge"))))
+ bpphs = archive.getAllPublishedBinaries()
+ # The binary publications are for the same binary package release,
+ # which has a single file attached to it.
+ self.assertEqual(1, len({bpph.binarypackagename for bpph in bpphs}))
+ self.assertEqual(1, len({bpph.binarypackagerelease for bpph in bpphs}))
+ self.assertThat(bpphs, MatchesSetwise(*(
+ MatchesStructure(
+ binarypackagename=MatchesStructure.byEquality(
+ name="wheel-indep"),
+ binarypackagerelease=MatchesStructure(
+ ci_build=Equals(build),
+ files=MatchesSetwise(
+ MatchesStructure.byEquality(
+ libraryfile=artifacts[1].library_file,
+ filetype=BinaryPackageFileType.WHL))),
+ binarypackageformat=Equals(BinaryPackageFormat.WHL),
+ distroarchseries=Equals(das),
+ channel=Equals(channel))
+ for channel in ("edge", "0.0.1/edge"))))
+
+ def test_existing_binary_release_no_existing_source_release(self):
+ # A `CIBuildUploadJob` can be run even if the build in question was
+ # already uploaded somewhere, and in that case may add publications
+ # in other locations for the same package. This works even if there
+ # was no existing source package release (because
+ # `CIBuildUploadJob`s didn't always create one).
+ archive = self.factory.makeArchive(
+ repository_format=ArchiveRepositoryFormat.PYTHON)
+ distroseries = self.factory.makeDistroSeries(
+ distribution=archive.distribution)
+ das = self.factory.makeDistroArchSeries(distroseries=distroseries)
+ build = self.makeCIBuild(archive.distribution, distro_arch_series=das)
+ report = build.getOrCreateRevisionStatusReport("build:0")
+ sdist_path = "wheel-indep/dist/wheel-indep-0.0.1.tar.gz"
+ with open(datadir(sdist_path), mode="rb") as f:
+ report.attach(name=os.path.basename(sdist_path), data=f.read())
+ wheel_path = "wheel-indep/dist/wheel_indep-0.0.1-py3-none-any.whl"
+ with open(datadir(wheel_path), mode="rb") as f:
+ report.attach(name=os.path.basename(wheel_path), data=f.read())
+ artifacts = IStore(RevisionStatusArtifact).find(
+ RevisionStatusArtifact,
+ report=report,
+ artifact_type=RevisionStatusArtifactType.BINARY).order_by("id")
+ job = CIBuildUploadJob.create(
+ build, build.git_repository.owner, archive, distroseries,
+ PackagePublishingPocket.RELEASE, target_channel="edge")
+ transaction.commit()
+ with MockPatchObject(CIBuildUploadJob, "_uploadSources"):
+ with dbuser(job.config.dbuser):
+ JobRunner([job]).runAll()
+ job = CIBuildUploadJob.create(
+ build, build.git_repository.owner, archive, distroseries,
+ PackagePublishingPocket.RELEASE, target_channel="0.0.1/edge")
+ transaction.commit()
+
+ with dbuser(job.config.dbuser):
+ JobRunner([job]).runAll()
+
+ # There is a source publication for a new source package release.
+ self.assertThat(archive.getPublishedSources(), MatchesSetwise(
+ MatchesStructure(
+ sourcepackagename=MatchesStructure.byEquality(
+ name=build.git_repository.target.name),
+ sourcepackagerelease=MatchesStructure(
+ ci_build=Equals(build),
+ files=MatchesSetwise(
+ MatchesStructure.byEquality(
+ libraryfile=artifacts[0].library_file,
+ filetype=SourcePackageFileType.SDIST))),
+ format=Equals(SourcePackageType.CI_BUILD),
+ distroseries=Equals(distroseries),
+ channel=Equals("0.0.1/edge"))))
bpphs = archive.getAllPublishedBinaries()
- # The publications are for the same binary package release, which
- # has a single file attached to it.
+ # The binary publications are for the same binary package release,
+ # which has a single file attached to it.
self.assertEqual(1, len({bpph.binarypackagename for bpph in bpphs}))
self.assertEqual(1, len({bpph.binarypackagerelease for bpph in bpphs}))
- self.assertThat(archive.getAllPublishedBinaries(), MatchesSetwise(*(
+ self.assertThat(bpphs, MatchesSetwise(*(
MatchesStructure(
binarypackagename=MatchesStructure.byEquality(
name="wheel-indep"),
@@ -595,7 +784,7 @@ class TestCIBuildUploadJob(TestCaseWithFactory):
ci_build=Equals(build),
files=MatchesSetwise(
MatchesStructure.byEquality(
- libraryfile=artifact.library_file,
+ libraryfile=artifacts[1].library_file,
filetype=BinaryPackageFileType.WHL))),
binarypackageformat=Equals(BinaryPackageFormat.WHL),
distroarchseries=Equals(das),
