launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #28485
[Merge] ~cjwatson/launchpad:ci-build-upload-job into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:ci-build-upload-job into launchpad:master.
Commit message:
Add method to upload a CI build to an archive
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/423198
This new `Archive.uploadCIBuild` method will be used to publish the output of CI builds in archives that publish via Artifactory. It's somewhat similar to `Archive.copyPackage`, and I considered calling it `copyCIBuild` instead, but I decided to go with "upload" since it's only usable for the initial copy into an archive, not for copies between archives after that.
There's a new `CIBuildUploadJob` which fetches the output files from the librarian and works out how to turn them into `BinaryPackageRelease`s. This is inevitably package-type-specific, and for the time being is only implemented for Python wheels, using `pkginfo` and `wheel-filename`.
Dependencies MP: https://code.launchpad.net/~cjwatson/lp-source-dependencies/+git/lp-source-dependencies/+merge/423187
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:ci-build-upload-job into launchpad:master.
diff --git a/database/schema/security.cfg b/database/schema/security.cfg
index 0024832..f04f672 100644
--- a/database/schema/security.cfg
+++ b/database/schema/security.cfg
@@ -1433,7 +1433,7 @@ public.archivesigningkey = SELECT, INSERT
public.binarypackagebuild = SELECT, INSERT, UPDATE
public.binarypackagefile = SELECT, INSERT
public.binarypackagename = SELECT, INSERT
-public.binarypackagepublishinghistory = SELECT
+public.binarypackagepublishinghistory = SELECT, INSERT
public.binarypackagerelease = SELECT, INSERT
public.binarysourcereference = SELECT, INSERT
public.bug = SELECT, UPDATE
diff --git a/lib/lp/_schema_circular_imports.py b/lib/lp/_schema_circular_imports.py
index 1d3cb17..6bf443b 100644
--- a/lib/lp/_schema_circular_imports.py
+++ b/lib/lp/_schema_circular_imports.py
@@ -397,6 +397,7 @@ patch_plain_parameter_type(IArchive, 'syncSource', 'from_archive', IArchive)
patch_plain_parameter_type(IArchive, 'copyPackage', 'from_archive', IArchive)
patch_plain_parameter_type(
IArchive, 'copyPackages', 'from_archive', IArchive)
+patch_plain_parameter_type(IArchive, 'uploadCIBuild', 'ci_build', ICIBuild)
patch_entry_return_type(IArchive, 'newSubscription', IArchiveSubscriber)
patch_plain_parameter_type(
IArchive, 'getArchiveDependency', 'dependency', IArchive)
diff --git a/lib/lp/services/config/schema-lazr.conf b/lib/lp/services/config/schema-lazr.conf
index 45bf05a..4fc8131 100644
--- a/lib/lp/services/config/schema-lazr.conf
+++ b/lib/lp/services/config/schema-lazr.conf
@@ -1914,6 +1914,7 @@ dbuser: process-job-source-groups
# can be loaded from.
job_sources:
IBranchModifiedMailJobSource,
+ ICIBuildUploadJobSource,
ICommercialExpiredJobSource,
IExpiringMembershipNotificationJobSource,
IGitRepositoryModifiedMailJobSource,
@@ -1966,6 +1967,11 @@ module: lp.charms.interfaces.charmrecipejob
dbuser: charm-build-job
crontab_group: MAIN
+[ICIBuildUploadJobSource]
+module: lp.soyuz.interfaces.archivejob
+dbuser: uploader
+crontab_group: FREQUENT
+
[ICommercialExpiredJobSource]
module: lp.registry.interfaces.productjob
dbuser: product-job
diff --git a/lib/lp/soyuz/configure.zcml b/lib/lp/soyuz/configure.zcml
index 29cabca..7236f92 100644
--- a/lib/lp/soyuz/configure.zcml
+++ b/lib/lp/soyuz/configure.zcml
@@ -433,21 +433,32 @@
<class class="lp.soyuz.model.archivejob.ArchiveJob">
<allow interface="lp.soyuz.interfaces.archivejob.IArchiveJob"/>
</class>
- <class class="lp.soyuz.model.archivejob.PackageUploadNotificationJob">
- <allow interface="lp.soyuz.interfaces.archivejob.IArchiveJob"/>
- <allow interface="lp.soyuz.interfaces.archivejob.IPackageUploadNotificationJob"/>
- </class>
<securedutility
component="lp.soyuz.model.archivejob.ArchiveJob"
provides="lp.soyuz.interfaces.archivejob.IArchiveJobSource">
<allow interface="lp.soyuz.interfaces.archivejob.IArchiveJobSource"/>
</securedutility>
+
+ <class class="lp.soyuz.model.archivejob.PackageUploadNotificationJob">
+ <allow interface="lp.soyuz.interfaces.archivejob.IArchiveJob"/>
+ <allow interface="lp.soyuz.interfaces.archivejob.IPackageUploadNotificationJob"/>
+ </class>
<securedutility
component="lp.soyuz.model.archivejob.PackageUploadNotificationJob"
provides="lp.soyuz.interfaces.archivejob.IPackageUploadNotificationJobSource">
<allow interface="lp.soyuz.interfaces.archivejob.IPackageUploadNotificationJobSource"/>
</securedutility>
+ <class class="lp.soyuz.model.archivejob.CIBuildUploadJob">
+ <allow interface="lp.soyuz.interfaces.archivejob.IArchiveJob"/>
+ <allow interface="lp.soyuz.interfaces.archivejob.ICIBuildUploadJob"/>
+ </class>
+ <securedutility
+ component="lp.soyuz.model.archivejob.CIBuildUploadJob"
+ provides="lp.soyuz.interfaces.archivejob.ICIBuildUploadJobSource">
+ <allow interface="lp.soyuz.interfaces.archivejob.ICIBuildUploadJobSource"/>
+ </securedutility>
+
<!-- ArchivePermission -->
<class
diff --git a/lib/lp/soyuz/enums.py b/lib/lp/soyuz/enums.py
index 9953f43..39b8ceb 100644
--- a/lib/lp/soyuz/enums.py
+++ b/lib/lp/soyuz/enums.py
@@ -57,6 +57,12 @@ class ArchiveJobType(DBEnumeratedType):
or held for approval.
""")
+ CI_BUILD_UPLOAD = DBItem(2, """
+ CI build upload
+
+ Upload a CI build to this archive.
+ """)
+
class ArchivePermissionType(DBEnumeratedType):
"""Archive Permission Type.
diff --git a/lib/lp/soyuz/interfaces/archive.py b/lib/lp/soyuz/interfaces/archive.py
index e011591..5ee645a 100644
--- a/lib/lp/soyuz/interfaces/archive.py
+++ b/lib/lp/soyuz/interfaces/archive.py
@@ -1672,6 +1672,18 @@ class IArchiveView(IHasBuildRecords):
:raises CannotCopy: if there is a problem copying.
"""
+ @call_with(person=REQUEST_USER)
+ @operation_parameters(
+ # Really ICIBuild, patched in _schema_circular_imports.
+ ci_build=Reference(schema=Interface),
+ to_series=TextLine(title=_("Target distroseries name")),
+ to_pocket=TextLine(title=_("Target pocket name")),
+ to_channel=TextLine(title=_("Target channel"), required=False))
+ @export_write_operation()
+ @operation_for_version('devel')
+ def uploadCIBuild(ci_build, person, to_series, to_pocket, to_channel=None):
+ """Upload the output of a CI build to this archive."""
+
class IArchiveAppend(Interface):
"""Archive interface for operations restricted by append privilege."""
diff --git a/lib/lp/soyuz/interfaces/archivejob.py b/lib/lp/soyuz/interfaces/archivejob.py
index 779bc0b..ae25b08 100644
--- a/lib/lp/soyuz/interfaces/archivejob.py
+++ b/lib/lp/soyuz/interfaces/archivejob.py
@@ -6,21 +6,29 @@
__all__ = [
'IArchiveJob',
'IArchiveJobSource',
+ 'ICIBuildUploadJob',
+ 'ICIBuildUploadJobSource',
'IPackageUploadNotificationJob',
'IPackageUploadNotificationJobSource',
]
+from lazr.restful.fields import Reference
from zope.interface import (
Attribute,
Interface,
)
from zope.schema import (
+ Choice,
Int,
Object,
+ TextLine,
)
from lp import _
+from lp.code.interfaces.cibuild import ICIBuild
+from lp.registry.interfaces.distroseries import IDistroSeries
+from lp.registry.interfaces.pocket import PackagePublishingPocket
from lp.services.job.interfaces.job import (
IJob,
IJobSource,
@@ -62,3 +70,26 @@ class IPackageUploadNotificationJob(IRunnableJob):
class IPackageUploadNotificationJobSource(IArchiveJobSource):
"""Interface for acquiring PackageUploadNotificationJobs."""
+
+
+class ICIBuildUploadJob(IRunnableJob):
+ """A Job to upload a CI build to an archive."""
+
+ ci_build = Reference(
+ schema=ICIBuild, title=_("CI build to copy"),
+ required=True, readonly=True)
+
+ target_distroseries = Reference(
+ schema=IDistroSeries, title=_("Target distroseries"),
+ required=True, readonly=True)
+
+ target_pocket = Choice(
+ title=_("Target pocket"), vocabulary=PackagePublishingPocket,
+ required=True, readonly=True)
+
+ target_channel = TextLine(
+ title=_("Target channel"), required=False, readonly=True)
+
+
+class ICIBuildUploadJobSource(IArchiveJobSource):
+ """Interface for acquiring `CIBuildUploadJob`s."""
diff --git a/lib/lp/soyuz/model/archive.py b/lib/lp/soyuz/model/archive.py
index 727d8b2..4d1ccb1 100644
--- a/lib/lp/soyuz/model/archive.py
+++ b/lib/lp/soyuz/model/archive.py
@@ -187,6 +187,7 @@ from lp.soyuz.interfaces.archive import (
VersionRequiresName,
)
from lp.soyuz.interfaces.archiveauthtoken import IArchiveAuthTokenSet
+from lp.soyuz.interfaces.archivejob import ICIBuildUploadJobSource
from lp.soyuz.interfaces.archivepermission import IArchivePermissionSet
from lp.soyuz.interfaces.archivesubscriber import (
ArchiveSubscriptionError,
@@ -1990,6 +1991,34 @@ class Archive(SQLBase):
sources, self, series, pocket, include_binaries, person=person,
check_permissions=False, unembargo=True)
+ def uploadCIBuild(self, ci_build, person, to_series, to_pocket,
+ to_channel=None):
+ """See `IArchive`."""
+ series = self._text_to_series(to_series)
+ pocket = self._text_to_pocket(to_pocket)
+ if self.publishing_method != ArchivePublishingMethod.ARTIFACTORY:
+ raise CannotCopy(
+ "CI builds may only be uploaded to archives published using "
+ "Artifactory.")
+ if ci_build.status != BuildStatus.FULLYBUILT:
+ raise CannotCopy(
+ "%r has status '%s', not '%s'." %
+ (ci_build, ci_build.status.title, BuildStatus.FULLYBUILT))
+ # Check upload permissions. We don't know the package name until we
+ # actually run the job; however, per-package upload permissions are
+ # by source package name, so don't necessarily make sense for CI
+ # builds anyway. For now, just ignore per-package upload
+ # permissions.
+ reason = self.checkUpload(
+ person=person, distroseries=series, sourcepackagename=None,
+ component=None, pocket=pocket)
+ if reason is not None:
+ raise CannotCopy(reason)
+ getUtility(ICIBuildUploadJobSource).create(
+ ci_build=ci_build, requester=person, target_archive=self,
+ target_distroseries=series, target_pocket=pocket,
+ target_channel=to_channel)
+
def getAuthToken(self, person):
"""See `IArchive`."""
diff --git a/lib/lp/soyuz/model/archivejob.py b/lib/lp/soyuz/model/archivejob.py
index e6785c6..eb918c1 100644
--- a/lib/lp/soyuz/model/archivejob.py
+++ b/lib/lp/soyuz/model/archivejob.py
@@ -3,20 +3,29 @@
import io
import logging
+import os.path
+import tempfile
from lazr.delegates import delegate_to
+from pkginfo import Wheel
from storm.expr import And
from storm.locals import (
Int,
JSON,
Reference,
)
+from wheel_filename import parse_wheel_filename
from zope.component import getUtility
from zope.interface import (
implementer,
provider,
)
+from lp.code.enums import RevisionStatusArtifactType
+from lp.code.interfaces.cibuild import ICIBuildSet
+from lp.code.interfaces.revisionstatus import IRevisionStatusArtifactSet
+from lp.registry.interfaces.distroseries import IDistroSeriesSet
+from lp.registry.interfaces.pocket import PackagePublishingPocket
from lp.services.config import config
from lp.services.database.enumcol import DBEnum
from lp.services.database.interfaces import IMasterStore
@@ -26,16 +35,22 @@ from lp.services.job.model.job import (
Job,
)
from lp.services.job.runner import BaseRunnableJob
+from lp.services.librarian.utils import copy_and_close
from lp.soyuz.enums import (
ArchiveJobType,
+ BinaryPackageFormat,
PackageUploadStatus,
)
from lp.soyuz.interfaces.archivejob import (
IArchiveJob,
IArchiveJobSource,
+ ICIBuildUploadJob,
+ ICIBuildUploadJobSource,
IPackageUploadNotificationJob,
IPackageUploadNotificationJobSource,
)
+from lp.soyuz.interfaces.binarypackagename import IBinaryPackageNameSet
+from lp.soyuz.interfaces.publishing import IPublishingSet
from lp.soyuz.interfaces.queue import IPackageUploadSet
from lp.soyuz.model.archive import Archive
@@ -165,3 +180,115 @@ class PackageUploadNotificationJob(ArchiveJobDerived):
packageupload.notify(
status=self.packageupload_status, summary_text=self.summary_text,
changes_file_object=changes_file_object, logger=logger)
+
+
+class ScanException(Exception):
+ """A CI build upload job failed to scan a file."""
+
+
+@implementer(ICIBuildUploadJob)
+@provider(ICIBuildUploadJobSource)
+class CIBuildUploadJob(ArchiveJobDerived):
+
+ class_job_type = ArchiveJobType.CI_BUILD_UPLOAD
+
+ config = config.ICIBuildUploadJobSource
+
+ @classmethod
+ def create(cls, ci_build, requester, target_archive, target_distroseries,
+ target_pocket, target_channel=None):
+ """See `ICIBuildUploadJobSource`."""
+ metadata = {
+ "ci_build_id": ci_build.id,
+ "target_distroseries_id": target_distroseries.id,
+ "target_pocket": target_pocket.title,
+ "target_channel": target_channel,
+ }
+ derived = super().create(target_archive, metadata)
+ derived.job.requester = requester
+ return derived
+
+ def getOopsVars(self):
+ vars = super().getOopsVars()
+ vars.extend([
+ (key, self.metadata[key])
+ for key in (
+ "ci_build_id",
+ "target_distroseries_id",
+ "target_pocket",
+ "target_channel",
+ )])
+ return vars
+
+ @property
+ def ci_build(self):
+ return getUtility(ICIBuildSet).getByID(self.metadata["ci_build_id"])
+
+ @property
+ def target_distroseries(self):
+ return getUtility(IDistroSeriesSet).get(
+ self.metadata["target_distroseries_id"])
+
+ @property
+ def target_pocket(self):
+ return PackagePublishingPocket.getTermByToken(
+ self.metadata["target_pocket"]).value
+
+ @property
+ def target_channel(self):
+ return self.metadata["target_channel"]
+
+ def _scanFile(self, path):
+ if path.endswith(".whl"):
+ try:
+ parsed_path = parse_wheel_filename(path)
+ wheel = Wheel(path)
+ except Exception as e:
+ raise ScanException("Failed to scan %s" % path) from e
+ return {
+ "name": wheel.name,
+ "version": wheel.version,
+ "summary": wheel.summary or "",
+ "description": wheel.description,
+ "binpackageformat": BinaryPackageFormat.WHL,
+ "architecturespecific": "any" not in parsed_path.platform_tags,
+ "homepage": wheel.home_page or "",
+ }
+ else:
+ return None
+
+ def run(self):
+ """See `IRunnableJob`."""
+ logger = logging.getLogger()
+ with tempfile.TemporaryDirectory(prefix="ci-build-copy-job") as tmpdir:
+ binaries = {}
+ for artifact in getUtility(
+ IRevisionStatusArtifactSet).findByCIBuild(self.ci_build):
+ if artifact.artifact_type == RevisionStatusArtifactType.LOG:
+ continue
+ name = artifact.library_file.filename
+ contents = os.path.join(tmpdir, name)
+ artifact.library_file.open()
+ copy_and_close(artifact.library_file, open(contents, "wb"))
+ metadata = self._scanFile(contents)
+ if metadata is None:
+ logger.info("No upload handler for %s" % name)
+ continue
+ logger.info(
+ "Uploading %s to %s %s (%s)" % (
+ name, self.archive.reference,
+ self.target_distroseries.getSuite(self.target_pocket),
+ self.target_channel))
+ metadata["binarypackagename"] = (
+ getUtility(IBinaryPackageNameSet).ensure(metadata["name"]))
+ del metadata["name"]
+ bpr = self.ci_build.createBinaryPackageRelease(**metadata)
+ bpr.addFile(artifact.library_file)
+ # The publishBinaries interface was designed for .debs,
+ # which need extra per-binary "override" information
+ # (component, etc.). None of this is relevant here.
+ binaries[bpr] = (None, None, None, None)
+ if binaries:
+ getUtility(IPublishingSet).publishBinaries(
+ self.archive, self.target_distroseries, self.target_pocket,
+ binaries, channel=self.target_channel)
diff --git a/lib/lp/soyuz/model/publishing.py b/lib/lp/soyuz/model/publishing.py
index 8a397b6..b229c47 100644
--- a/lib/lp/soyuz/model/publishing.py
+++ b/lib/lp/soyuz/model/publishing.py
@@ -1097,7 +1097,7 @@ def expand_binary_requests(distroseries, binaries):
# Find the DAS in this series corresponding to the original
# build arch tag. If it does not exist or is disabled, we should
# not publish.
- target_arch = arch_map.get(bpr.build.arch_tag)
+ target_arch = arch_map.get((bpr.build or bpr.ci_build).arch_tag)
target_archs = [target_arch] if target_arch is not None else []
else:
target_archs = archs
diff --git a/lib/lp/soyuz/tests/__init__.py b/lib/lp/soyuz/tests/__init__.py
index e69de29..7ffe3f9 100644
--- a/lib/lp/soyuz/tests/__init__.py
+++ b/lib/lp/soyuz/tests/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2022 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+import os
+
+
+here = os.path.dirname(os.path.realpath(__file__))
+
+
+def datadir(path):
+ """Return fully-qualified path inside the test data directory."""
+ if path.startswith("/"):
+ raise ValueError("Path is not relative: %s" % path)
+ return os.path.join(here, "data", path)
diff --git a/lib/lp/soyuz/tests/data/.gitignore b/lib/lp/soyuz/tests/data/.gitignore
new file mode 100644
index 0000000..2955230
--- /dev/null
+++ b/lib/lp/soyuz/tests/data/.gitignore
@@ -0,0 +1,2 @@
+!dist
+*.egg-info
diff --git a/lib/lp/soyuz/tests/data/wheel-arch/README.md b/lib/lp/soyuz/tests/data/wheel-arch/README.md
new file mode 100644
index 0000000..3135dc3
--- /dev/null
+++ b/lib/lp/soyuz/tests/data/wheel-arch/README.md
@@ -0,0 +1 @@
+An example package. Build a wheel from this with `pyproject-build`.
diff --git a/lib/lp/soyuz/tests/data/wheel-arch/dist/wheel-arch-0.0.1.tar.gz b/lib/lp/soyuz/tests/data/wheel-arch/dist/wheel-arch-0.0.1.tar.gz
new file mode 100644
index 0000000..5b61840
Binary files /dev/null and b/lib/lp/soyuz/tests/data/wheel-arch/dist/wheel-arch-0.0.1.tar.gz differ
diff --git a/lib/lp/soyuz/tests/data/wheel-arch/dist/wheel_arch-0.0.1-cp310-cp310-linux_x86_64.whl b/lib/lp/soyuz/tests/data/wheel-arch/dist/wheel_arch-0.0.1-cp310-cp310-linux_x86_64.whl
new file mode 100644
index 0000000..2131f4d
Binary files /dev/null and b/lib/lp/soyuz/tests/data/wheel-arch/dist/wheel_arch-0.0.1-cp310-cp310-linux_x86_64.whl differ
diff --git a/lib/lp/soyuz/tests/data/wheel-arch/pyproject.toml b/lib/lp/soyuz/tests/data/wheel-arch/pyproject.toml
new file mode 100644
index 0000000..b0f0765
--- /dev/null
+++ b/lib/lp/soyuz/tests/data/wheel-arch/pyproject.toml
@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools>=42"]
+build-backend = "setuptools.build_meta"
diff --git a/lib/lp/soyuz/tests/data/wheel-arch/setup.py b/lib/lp/soyuz/tests/data/wheel-arch/setup.py
new file mode 100755
index 0000000..d1431d7
--- /dev/null
+++ b/lib/lp/soyuz/tests/data/wheel-arch/setup.py
@@ -0,0 +1,14 @@
+from setuptools import (
+ Extension,
+ setup,
+ )
+
+
+setup(
+ name="wheel-arch",
+ version="0.0.1",
+ description="Example description",
+ long_description="Example long description",
+ url="http://example.com/";,
+ ext_modules=[Extension("_test", sources=["test.c"])],
+ )
diff --git a/lib/lp/soyuz/tests/data/wheel-arch/test.c b/lib/lp/soyuz/tests/data/wheel-arch/test.c
new file mode 100644
index 0000000..576fc6d
--- /dev/null
+++ b/lib/lp/soyuz/tests/data/wheel-arch/test.c
@@ -0,0 +1 @@
+#include <Python.h>
diff --git a/lib/lp/soyuz/tests/data/wheel-indep/README.md b/lib/lp/soyuz/tests/data/wheel-indep/README.md
new file mode 100644
index 0000000..3135dc3
--- /dev/null
+++ b/lib/lp/soyuz/tests/data/wheel-indep/README.md
@@ -0,0 +1 @@
+An example package. Build a wheel from this with `pyproject-build`.
diff --git a/lib/lp/soyuz/tests/data/wheel-indep/dist/wheel-indep-0.0.1.tar.gz b/lib/lp/soyuz/tests/data/wheel-indep/dist/wheel-indep-0.0.1.tar.gz
new file mode 100644
index 0000000..16fc042
Binary files /dev/null and b/lib/lp/soyuz/tests/data/wheel-indep/dist/wheel-indep-0.0.1.tar.gz differ
diff --git a/lib/lp/soyuz/tests/data/wheel-indep/dist/wheel_indep-0.0.1-py3-none-any.whl b/lib/lp/soyuz/tests/data/wheel-indep/dist/wheel_indep-0.0.1-py3-none-any.whl
new file mode 100644
index 0000000..93b54bd
Binary files /dev/null and b/lib/lp/soyuz/tests/data/wheel-indep/dist/wheel_indep-0.0.1-py3-none-any.whl differ
diff --git a/lib/lp/soyuz/tests/data/wheel-indep/pyproject.toml b/lib/lp/soyuz/tests/data/wheel-indep/pyproject.toml
new file mode 100644
index 0000000..b0f0765
--- /dev/null
+++ b/lib/lp/soyuz/tests/data/wheel-indep/pyproject.toml
@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools>=42"]
+build-backend = "setuptools.build_meta"
diff --git a/lib/lp/soyuz/tests/data/wheel-indep/setup.cfg b/lib/lp/soyuz/tests/data/wheel-indep/setup.cfg
new file mode 100644
index 0000000..ae136d3
--- /dev/null
+++ b/lib/lp/soyuz/tests/data/wheel-indep/setup.cfg
@@ -0,0 +1,5 @@
+[metadata]
+name = wheel-indep
+version = 0.0.1
+description = Example description
+long_description = Example long description
diff --git a/lib/lp/soyuz/tests/test_archive.py b/lib/lp/soyuz/tests/test_archive.py
index a02d75d..fd3d254 100644
--- a/lib/lp/soyuz/tests/test_archive.py
+++ b/lib/lp/soyuz/tests/test_archive.py
@@ -88,6 +88,7 @@ from lp.soyuz.adapters.overrides import (
)
from lp.soyuz.enums import (
ArchivePermissionType,
+ ArchivePublishingMethod,
ArchivePurpose,
ArchiveStatus,
PackageCopyPolicy,
@@ -116,6 +117,7 @@ from lp.soyuz.interfaces.archive import (
RedirectedPocket,
VersionRequiresName,
)
+from lp.soyuz.interfaces.archivejob import ICIBuildUploadJobSource
from lp.soyuz.interfaces.archivepermission import IArchivePermissionSet
from lp.soyuz.interfaces.binarypackagebuild import BuildSetStatus
from lp.soyuz.interfaces.binarypackagename import IBinaryPackageNameSet
@@ -3446,6 +3448,71 @@ class TestCopyPackage(TestCaseWithFactory):
person=source_archive.owner, move=True)
+class TestUploadCIBuild(TestCaseWithFactory):
+
+ layer = DatabaseFunctionalLayer
+
+ def test_creates_job(self):
+ # The uploadCIBuild method creates a CIBuildUploadJob with the
+ # appropriate parameters.
+ archive = self.factory.makeArchive(
+ publishing_method=ArchivePublishingMethod.ARTIFACTORY)
+ series = self.factory.makeDistroSeries(
+ distribution=archive.distribution)
+ build = self.factory.makeCIBuild(status=BuildStatus.FULLYBUILT)
+ with person_logged_in(archive.owner):
+ archive.uploadCIBuild(
+ build, archive.owner, series.name, "Release",
+ to_channel="edge")
+ [job] = getUtility(ICIBuildUploadJobSource).iterReady()
+ self.assertThat(job, MatchesStructure.byEquality(
+ ci_build=build,
+ target_distroseries=series,
+ target_pocket=PackagePublishingPocket.RELEASE,
+ target_channel="edge"))
+
+ def test_disallows_non_artifactory_publishing(self):
+ # CI builds may only be copied into archives published using
+ # Artifactory.
+ archive = self.factory.makeArchive()
+ series = self.factory.makeDistroSeries(
+ distribution=archive.distribution)
+ build = self.factory.makeCIBuild(status=BuildStatus.FULLYBUILT)
+ with person_logged_in(archive.owner):
+ self.assertRaisesWithContent(
+ CannotCopy,
+ "CI builds may only be uploaded to archives published using "
+ "Artifactory.",
+ archive.uploadCIBuild,
+ build, archive.owner, series.name, "Release")
+
+ def test_disallows_incomplete_builds(self):
+ # CI builds with statuses other than FULLYBUILT may not be copied.
+ archive = self.factory.makeArchive(
+ publishing_method=ArchivePublishingMethod.ARTIFACTORY)
+ series = self.factory.makeDistroSeries(
+ distribution=archive.distribution)
+ build = self.factory.makeCIBuild(status=BuildStatus.FAILEDTOBUILD)
+ person = self.factory.makePerson()
+ self.assertRaisesWithContent(
+ CannotCopy,
+ "%r has status 'Failed to build', not 'Successfully built'." % (
+ build),
+ archive.uploadCIBuild, build, person, series.name, "Release")
+
+ def test_disallows_non_uploaders(self):
+ # Only people with upload permission may call uploadCIBuild.
+ archive = self.factory.makeArchive(
+ publishing_method=ArchivePublishingMethod.ARTIFACTORY)
+ series = self.factory.makeDistroSeries(
+ distribution=archive.distribution)
+ build = self.factory.makeCIBuild(status=BuildStatus.FULLYBUILT)
+ person = self.factory.makePerson()
+ self.assertRaisesWithContent(
+ CannotCopy, "Signer has no upload rights to this PPA.",
+ archive.uploadCIBuild, build, person, series.name, "Release")
+
+
class TestgetAllPublishedBinaries(TestCaseWithFactory):
layer = DatabaseFunctionalLayer
diff --git a/lib/lp/soyuz/tests/test_archivejob.py b/lib/lp/soyuz/tests/test_archivejob.py
index f00e606..e6c1c28 100644
--- a/lib/lp/soyuz/tests/test_archivejob.py
+++ b/lib/lp/soyuz/tests/test_archivejob.py
@@ -1,19 +1,36 @@
# Copyright 2010-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
+import os.path
+
from debian.deb822 import Changes
+from testtools.matchers import (
+ Equals,
+ Is,
+ MatchesSetwise,
+ MatchesStructure,
+ )
+import transaction
+from lp.code.enums import RevisionStatusArtifactType
+from lp.code.model.revisionstatus import RevisionStatusArtifact
+from lp.registry.interfaces.pocket import PackagePublishingPocket
+from lp.services.database.interfaces import IStore
from lp.services.job.runner import JobRunner
from lp.services.mail.sendmail import format_address_for_person
from lp.soyuz.enums import (
ArchiveJobType,
+ BinaryPackageFileType,
+ BinaryPackageFormat,
PackageUploadStatus,
)
from lp.soyuz.model.archivejob import (
ArchiveJob,
ArchiveJobDerived,
+ CIBuildUploadJob,
PackageUploadNotificationJob,
)
+from lp.soyuz.tests import datadir
from lp.testing import TestCaseWithFactory
from lp.testing.dbuser import dbuser
from lp.testing.layers import (
@@ -117,3 +134,181 @@ class TestPackageUploadNotificationJob(TestCaseWithFactory):
self.assertEqual(format_address_for_person(creator), email['To'])
self.assertIn('(Accepted)', email['Subject'])
self.assertIn('Fake summary', email.get_payload()[0].get_payload())
+
+
+class TestCIBuildUploadJob(TestCaseWithFactory):
+
+ layer = LaunchpadZopelessLayer
+
+ def test_getOopsVars(self):
+ archive = self.factory.makeArchive()
+ distroseries = self.factory.makeDistroSeries(
+ distribution=archive.distribution)
+ build = self.factory.makeCIBuild()
+ job = CIBuildUploadJob.create(
+ build, build.git_repository.owner, archive, distroseries,
+ PackagePublishingPocket.RELEASE, target_channel="edge")
+ expected = [
+ ("job_id", job.context.job.id),
+ ("archive_id", archive.id),
+ ("archive_job_id", job.context.id),
+ ("archive_job_type", "CI build upload"),
+ ("ci_build_id", build.id),
+ ("target_distroseries_id", distroseries.id),
+ ("target_pocket", "Release"),
+ ("target_channel", "edge"),
+ ]
+ self.assertEqual(expected, job.getOopsVars())
+
+ def test_metadata(self):
+ archive = self.factory.makeArchive()
+ distroseries = self.factory.makeDistroSeries(
+ distribution=archive.distribution)
+ build = self.factory.makeCIBuild()
+ job = CIBuildUploadJob.create(
+ build, build.git_repository.owner, archive, distroseries,
+ PackagePublishingPocket.RELEASE, target_channel="edge")
+ expected = {
+ "ci_build_id": build.id,
+ "target_distroseries_id": distroseries.id,
+ "target_pocket": "Release",
+ "target_channel": "edge",
+ }
+ self.assertEqual(expected, job.metadata)
+ self.assertEqual(build, job.ci_build)
+ self.assertEqual(distroseries, job.target_distroseries)
+ self.assertEqual(PackagePublishingPocket.RELEASE, job.target_pocket)
+ self.assertEqual("edge", job.target_channel)
+
+ def test__scanFile_wheel_indep(self):
+ archive = self.factory.makeArchive()
+ distroseries = self.factory.makeDistroSeries(
+ distribution=archive.distribution)
+ build = self.factory.makeCIBuild()
+ job = CIBuildUploadJob.create(
+ build, build.git_repository.owner, archive, distroseries,
+ PackagePublishingPocket.RELEASE, target_channel="edge")
+ path = "wheel-indep/dist/wheel_indep-0.0.1-py3-none-any.whl"
+ expected = {
+ "name": "wheel-indep",
+ "version": "0.0.1",
+ "summary": "Example description",
+ "description": "Example long description\n",
+ "binpackageformat": BinaryPackageFormat.WHL,
+ "architecturespecific": False,
+ "homepage": "",
+ }
+ self.assertEqual(expected, job._scanFile(datadir(path)))
+
+ def test__scanFile_wheel_arch(self):
+ archive = self.factory.makeArchive()
+ distroseries = self.factory.makeDistroSeries(
+ distribution=archive.distribution)
+ build = self.factory.makeCIBuild()
+ job = CIBuildUploadJob.create(
+ build, build.git_repository.owner, archive, distroseries,
+ PackagePublishingPocket.RELEASE, target_channel="edge")
+ path = "wheel-arch/dist/wheel_arch-0.0.1-cp310-cp310-linux_x86_64.whl"
+ expected = {
+ "name": "wheel-arch",
+ "version": "0.0.1",
+ "summary": "Example description",
+ "description": "Example long description\n",
+ "binpackageformat": BinaryPackageFormat.WHL,
+ "architecturespecific": True,
+ "homepage": "http://example.com/";,
+ }
+ self.assertEqual(expected, job._scanFile(datadir(path)))
+
+ def test_run_indep(self):
+ archive = self.factory.makeArchive()
+ distroseries = self.factory.makeDistroSeries(
+ distribution=archive.distribution)
+ dases = [
+ self.factory.makeDistroArchSeries(distroseries=distroseries)
+ for _ in range(2)]
+ build = self.factory.makeCIBuild(distro_arch_series=dases[0])
+ report = build.getOrCreateRevisionStatusReport("build:0")
+ report.setLog(b"log data")
+ path = "wheel-indep/dist/wheel_indep-0.0.1-py3-none-any.whl"
+ with open(datadir(path), mode="rb") as f:
+ report.attach(name=os.path.basename(path), data=f.read())
+ artifact = IStore(RevisionStatusArtifact).find(
+ RevisionStatusArtifact,
+ report=report,
+ artifact_type=RevisionStatusArtifactType.BINARY).one()
+ job = CIBuildUploadJob.create(
+ build, build.git_repository.owner, archive, distroseries,
+ PackagePublishingPocket.RELEASE, target_channel="edge")
+ transaction.commit()
+
+ with dbuser(job.config.dbuser):
+ JobRunner([job]).runAll()
+
+ self.assertThat(archive.getAllPublishedBinaries(), MatchesSetwise(*(
+ MatchesStructure(
+ binarypackagename=MatchesStructure.byEquality(
+ name="wheel-indep"),
+ binarypackagerelease=MatchesStructure(
+ ci_build=Equals(build),
+ binarypackagename=MatchesStructure.byEquality(
+ name="wheel-indep"),
+ version=Equals("0.0.1"),
+ summary=Equals("Example description"),
+ description=Equals("Example long description\n"),
+ binpackageformat=Equals(BinaryPackageFormat.WHL),
+ architecturespecific=Is(False),
+ homepage=Equals(""),
+ files=MatchesSetwise(
+ MatchesStructure.byEquality(
+ libraryfile=artifact.library_file,
+ filetype=BinaryPackageFileType.WHL))),
+ binarypackageformat=Equals(BinaryPackageFormat.WHL),
+ distroarchseries=Equals(das))
+ for das in dases)))
+
+ def test_run_arch(self):
+ archive = self.factory.makeArchive()
+ distroseries = self.factory.makeDistroSeries(
+ distribution=archive.distribution)
+ dases = [
+ self.factory.makeDistroArchSeries(distroseries=distroseries)
+ for _ in range(2)]
+ build = self.factory.makeCIBuild(distro_arch_series=dases[0])
+ report = build.getOrCreateRevisionStatusReport("build:0")
+ report.setLog(b"log data")
+ path = "wheel-arch/dist/wheel_arch-0.0.1-cp310-cp310-linux_x86_64.whl"
+ with open(datadir(path), mode="rb") as f:
+ report.attach(name=os.path.basename(path), data=f.read())
+ artifact = IStore(RevisionStatusArtifact).find(
+ RevisionStatusArtifact,
+ report=report,
+ artifact_type=RevisionStatusArtifactType.BINARY).one()
+ job = CIBuildUploadJob.create(
+ build, build.git_repository.owner, archive, distroseries,
+ PackagePublishingPocket.RELEASE, target_channel="edge")
+ transaction.commit()
+
+ with dbuser(job.config.dbuser):
+ JobRunner([job]).runAll()
+
+ self.assertThat(archive.getAllPublishedBinaries(), MatchesSetwise(
+ MatchesStructure(
+ binarypackagename=MatchesStructure.byEquality(
+ name="wheel-arch"),
+ binarypackagerelease=MatchesStructure(
+ ci_build=Equals(build),
+ binarypackagename=MatchesStructure.byEquality(
+ name="wheel-arch"),
+ version=Equals("0.0.1"),
+ summary=Equals("Example description"),
+ description=Equals("Example long description\n"),
+ binpackageformat=Equals(BinaryPackageFormat.WHL),
+ architecturespecific=Is(True),
+ homepage=Equals("http://example.com/";),
+ files=MatchesSetwise(
+ MatchesStructure.byEquality(
+ libraryfile=artifact.library_file,
+ filetype=BinaryPackageFileType.WHL))),
+ binarypackageformat=Equals(BinaryPackageFormat.WHL),
+ distroarchseries=Equals(dases[0]))))
diff --git a/requirements/launchpad.txt b/requirements/launchpad.txt
index 981acfb..c2081c1 100644
--- a/requirements/launchpad.txt
+++ b/requirements/launchpad.txt
@@ -109,6 +109,7 @@ PasteDeploy==2.1.0
pathlib2==2.3.2
patiencediff==0.2.2
pgbouncer==0.0.9
+pkginfo==1.8.2
prettytable==0.7.2
psutil==5.4.2
psycopg2==2.8.6
@@ -174,6 +175,7 @@ webencodings==0.5.1
WebOb==1.8.5
WebTest==2.0.35
Werkzeug==1.0.1
+wheel-filename==1.1.0
wrapt==1.12.1
wsgi-intercept==1.9.2
WSGIProxy2==0.4.6
diff --git a/setup.cfg b/setup.cfg
index db4af9b..145a832 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -66,6 +66,7 @@ install_requires =
oops_twisted
oops_wsgi
paramiko
+ pkginfo
psutil
pgbouncer
psycopg2
@@ -110,6 +111,7 @@ install_requires =
WebOb
WebTest
Werkzeug
+ wheel-filename
WSGIProxy2
z3c.ptcompat
zope.app.http
