launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #19144
[Merge] lp:~cjwatson/launchpad/snap-build-behaviour into lp:launchpad
Colin Watson has proposed merging lp:~cjwatson/launchpad/snap-build-behaviour into lp:launchpad.
Commit message:
Add a build behaviour for snap packages.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
Related bugs:
Bug #1476405 in Launchpad itself: "Add support for building snaps"
https://bugs.launchpad.net/launchpad/+bug/1476405
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/snap-build-behaviour/+merge/266736
Add a build behaviour for snap packages.
This involves adding support for a build tools archive which can be set in a config variable for individual build behaviours, since snapcraft is currently only available in a PPA. We should eventually make this more fine-grained, but that isn't urgent.
You may want to review this alongside https://code.launchpad.net/~cjwatson/launchpad-buildd/snapcraft/+merge/266541, which implements the slave side of the builder protocol.
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/snap-build-behaviour into lp:launchpad.
=== modified file 'database/schema/security.cfg'
--- database/schema/security.cfg 2015-07-30 12:43:33 +0000
+++ database/schema/security.cfg 2015-08-03 14:55:59 +0000
@@ -967,6 +967,8 @@
public.distroseriesparent = SELECT
public.emailaddress = SELECT
public.flatpackagesetinclusion = SELECT
+public.gitref = SELECT
+public.gitrepository = SELECT
public.gpgkey = SELECT
public.libraryfilealias = SELECT, INSERT
public.libraryfilecontent = SELECT, INSERT
=== modified file 'lib/lp/services/config/schema-lazr.conf'
--- lib/lp/services/config/schema-lazr.conf 2015-08-03 08:18:09 +0000
+++ lib/lp/services/config/schema-lazr.conf 2015-08-03 14:55:59 +0000
@@ -1722,6 +1722,9 @@
# credentials, so the proxy needs to be very carefully secured.
http_proxy: none
+[snappy]
+tools_archive: none
+
[process-job-source-groups]
# This section is used by cronscripts/process-job-source-groups.py.
dbuser: process-job-source-groups
=== modified file 'lib/lp/snappy/configure.zcml'
--- lib/lp/snappy/configure.zcml 2015-07-30 15:19:51 +0000
+++ lib/lp/snappy/configure.zcml 2015-08-03 14:55:59 +0000
@@ -66,6 +66,13 @@
<allow interface="lp.buildmaster.interfaces.buildfarmjob.ISpecificBuildFarmJobSource" />
</securedutility>
+ <!-- SnapBuildBehaviour -->
+ <adapter
+ for="lp.snappy.interfaces.snapbuild.ISnapBuild"
+ provides="lp.buildmaster.interfaces.buildfarmjobbehaviour.IBuildFarmJobBehaviour"
+ factory="lp.snappy.model.snapbuildbehaviour.SnapBuildBehaviour"
+ permission="zope.Public" />
+
<!-- SnapFile -->
<class class="lp.snappy.model.snapbuild.SnapFile">
<allow interface="lp.snappy.interfaces.snapbuild.ISnapFile" />
=== added file 'lib/lp/snappy/model/snapbuildbehaviour.py'
--- lib/lp/snappy/model/snapbuildbehaviour.py 1970-01-01 00:00:00 +0000
+++ lib/lp/snappy/model/snapbuildbehaviour.py 2015-08-03 14:55:59 +0000
@@ -0,0 +1,117 @@
+# Copyright 2015 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""An `IBuildFarmJobBehaviour` for `SnapBuild`.
+
+Dispatches snap package build jobs to build-farm slaves.
+"""
+
+__metaclass__ = type
+__all__ = [
+ 'SnapBuildBehaviour',
+ ]
+
+from zope.component import (
+ adapter,
+ getUtility,
+ )
+from zope.interface import implementer
+
+from lp.buildmaster.interfaces.builder import CannotBuild
+from lp.buildmaster.interfaces.buildfarmjobbehaviour import (
+ IBuildFarmJobBehaviour,
+ )
+from lp.buildmaster.model.buildfarmjobbehaviour import (
+ BuildFarmJobBehaviourBase,
+ )
+from lp.registry.interfaces.series import SeriesStatus
+from lp.services.config import config
+from lp.snappy.interfaces.snap import SnapBuildArchiveOwnerMismatch
+from lp.snappy.interfaces.snapbuild import ISnapBuild
+from lp.soyuz.adapters.archivedependencies import (
+ get_sources_list_for_building,
+ )
+from lp.soyuz.interfaces.archive import (
+ ArchiveDisabled,
+ IArchiveSet,
+ )
+
+
+@adapter(ISnapBuild)
+@implementer(IBuildFarmJobBehaviour)
+class SnapBuildBehaviour(BuildFarmJobBehaviourBase):
+ """Dispatches `SnapBuild` jobs to slaves."""
+
+ def getLogFileName(self):
+ das = self.build.distro_arch_series
+
+ # Examples:
+ # buildlog_ubuntu_wily_amd64_name_FULLYBUILT.txt
+ return 'buildlog_%s_%s_%s_%s_%s.txt' % (
+ das.distroseries.distribution.name, das.distroseries.name,
+ das.architecturetag, self.build.snap.name, self.build.status.name)
+
+ def verifyBuildRequest(self, logger):
+ """Assert some pre-build checks.
+
+ The build request is checked:
+ * Virtualized builds can't build on a non-virtual builder
+ * The source archive may not be disabled
+ * If the source archive is private, the snap owner must match the
+ archive owner (see `SnapBuildArchiveOwnerMismatch` docstring)
+ * Ensure that we have a chroot
+ """
+ build = self.build
+ if build.virtualized and not self._builder.virtualized:
+ raise AssertionError(
+ "Attempt to build virtual item on a non-virtual builder.")
+
+ if not build.archive.enabled:
+ raise ArchiveDisabled(build.archive.displayname)
+ if build.archive.private and build.snap.owner != build.archive.owner:
+ raise SnapBuildArchiveOwnerMismatch()
+
+ chroot = build.distro_arch_series.getChroot()
+ if chroot is None:
+ raise CannotBuild(
+ "Missing chroot for %s" % build.distro_arch_series.displayname)
+
+ def _extraBuildArgs(self):
+ """
+ Return the extra arguments required by the slave for the given build.
+ """
+ build = self.build
+ args = {}
+ args["name"] = build.snap.name
+ args["arch_tag"] = build.distro_arch_series.architecturetag
+ # XXX cjwatson 2015-08-03: Allow this to be overridden at some more
+ # fine-grained level.
+ if config.snappy.tools_archive is not None:
+ tools_archive = getUtility(IArchiveSet).getByReference(
+ config.snappy.tools_archive)
+ else:
+ tools_archive = None
+ args["archives"] = get_sources_list_for_building(
+ build, build.distro_arch_series, None, tools_archive=tools_archive)
+ args["archive_private"] = build.archive.private
+ if build.snap.branch is not None:
+ args["branch"] = build.snap.branch.bzr_identity
+ else:
+ assert build.snap.git_repository is not None
+ args["git_repository"] = build.snap.git_repository.git_https_url
+ args["git_path"] = build.snap.git_path
+ return args
+
+ def composeBuildRequest(self, logger):
+ return (
+ "snap", self.build.distro_arch_series, {},
+ self._extraBuildArgs())
+
+ def verifySuccessfulBuild(self):
+ """See `IBuildFarmJobBehaviour`."""
+ # The implementation in BuildFarmJobBehaviourBase checks whether the
+ # target suite is modifiable in the target archive. However, a
+ # `SnapBuild`'s archive is a source rather than a target, so that
+ # check does not make sense. We do, however, refuse to build for
+ # obsolete series.
+ assert self.build.distro_series.status != SeriesStatus.OBSOLETE
=== added file 'lib/lp/snappy/tests/test_snapbuildbehaviour.py'
--- lib/lp/snappy/tests/test_snapbuildbehaviour.py 1970-01-01 00:00:00 +0000
+++ lib/lp/snappy/tests/test_snapbuildbehaviour.py 2015-08-03 14:55:59 +0000
@@ -0,0 +1,227 @@
+# Copyright 2015 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Test snap package build behaviour."""
+
+__metaclass__ = type
+
+import fixtures
+import transaction
+from twisted.trial.unittest import TestCase as TrialTestCase
+from zope.component import getUtility
+
+from lp.buildmaster.enums import BuildStatus
+from lp.buildmaster.interfaces.builder import CannotBuild
+from lp.buildmaster.interfaces.buildfarmjobbehaviour import (
+ IBuildFarmJobBehaviour,
+ )
+from lp.buildmaster.interfaces.processor import IProcessorSet
+from lp.buildmaster.tests.mock_slaves import (
+ MockBuilder,
+ OkSlave,
+ )
+from lp.buildmaster.tests.test_buildfarmjobbehaviour import (
+ TestGetUploadMethodsMixin,
+ TestHandleStatusMixin,
+ TestVerifySuccessfulBuildMixin,
+ )
+from lp.registry.interfaces.pocket import PackagePublishingPocket
+from lp.registry.interfaces.series import SeriesStatus
+from lp.services.features.testing import FeatureFixture
+from lp.services.log.logger import BufferLogger
+from lp.snappy.interfaces.snap import (
+ SNAP_FEATURE_FLAG,
+ SnapBuildArchiveOwnerMismatch,
+ )
+from lp.snappy.model.snapbuildbehaviour import SnapBuildBehaviour
+from lp.soyuz.adapters.archivedependencies import (
+ get_sources_list_for_building,
+ )
+from lp.soyuz.interfaces.archive import ArchiveDisabled
+from lp.testing import TestCaseWithFactory
+from lp.testing.layers import LaunchpadZopelessLayer
+
+
+class TestSnapBuildBehaviour(TestCaseWithFactory):
+
+ layer = LaunchpadZopelessLayer
+
+ def setUp(self):
+ super(TestSnapBuildBehaviour, self).setUp()
+ self.useFixture(FeatureFixture({SNAP_FEATURE_FLAG: u"on"}))
+
+ def makeJob(self, pocket=PackagePublishingPocket.RELEASE, **kwargs):
+ """Create a sample `ISnapBuildBehaviour`."""
+ distribution = self.factory.makeDistribution(name="distro")
+ distroseries = self.factory.makeDistroSeries(
+ distribution=distribution, name="unstable")
+ processor = getUtility(IProcessorSet).getByName("386")
+ distroarchseries = self.factory.makeDistroArchSeries(
+ distroseries=distroseries, architecturetag="i386",
+ processor=processor)
+ build = self.factory.makeSnapBuild(
+ distroarchseries=distroarchseries, pocket=pocket,
+ name=u"test-snap", **kwargs)
+ return IBuildFarmJobBehaviour(build)
+
+ def test_provides_interface(self):
+ # SnapBuildBehaviour provides IBuildFarmJobBehaviour.
+ job = SnapBuildBehaviour(None)
+ self.assertProvides(job, IBuildFarmJobBehaviour)
+
+ def test_adapts_ISnapBuild(self):
+ # IBuildFarmJobBehaviour adapts an ISnapBuild.
+ build = self.factory.makeSnapBuild()
+ job = IBuildFarmJobBehaviour(build)
+ self.assertProvides(job, IBuildFarmJobBehaviour)
+
+ def test_verifyBuildRequest_valid(self):
+ # verifyBuildRequest doesn't raise any exceptions when called with a
+ # valid builder set.
+ job = self.makeJob()
+ lfa = self.factory.makeLibraryFileAlias()
+ transaction.commit()
+ job.build.distro_arch_series.addOrUpdateChroot(lfa)
+ builder = MockBuilder()
+ job.setBuilder(builder, OkSlave())
+ logger = BufferLogger()
+ job.verifyBuildRequest(logger)
+ self.assertEqual("", logger.getLogBuffer())
+
+ def test_verifyBuildRequest_virtual_mismatch(self):
+ # verifyBuildRequest raises on an attempt to build a virtualized
+ # build on a non-virtual builder.
+ job = self.makeJob()
+ lfa = self.factory.makeLibraryFileAlias()
+ transaction.commit()
+ job.build.distro_arch_series.addOrUpdateChroot(lfa)
+ builder = MockBuilder(virtualized=False)
+ job.setBuilder(builder, OkSlave())
+ logger = BufferLogger()
+ e = self.assertRaises(AssertionError, job.verifyBuildRequest, logger)
+ self.assertEqual(
+ "Attempt to build virtual item on a non-virtual builder.", str(e))
+
+ def test_verifyBuildRequest_archive_disabled(self):
+ archive = self.factory.makeArchive(
+ enabled=False, displayname="Disabled Archive")
+ job = self.makeJob(archive=archive)
+ lfa = self.factory.makeLibraryFileAlias()
+ transaction.commit()
+ job.build.distro_arch_series.addOrUpdateChroot(lfa)
+ builder = MockBuilder()
+ job.setBuilder(builder, OkSlave())
+ logger = BufferLogger()
+ e = self.assertRaises(ArchiveDisabled, job.verifyBuildRequest, logger)
+ self.assertEqual("Disabled Archive is disabled.", str(e))
+
+ def test_verifyBuildRequest_archive_private_owners_match(self):
+ archive = self.factory.makeArchive(private=True)
+ job = self.makeJob(
+ archive=archive, registrant=archive.owner, owner=archive.owner)
+ lfa = self.factory.makeLibraryFileAlias()
+ transaction.commit()
+ job.build.distro_arch_series.addOrUpdateChroot(lfa)
+ builder = MockBuilder()
+ job.setBuilder(builder, OkSlave())
+ logger = BufferLogger()
+ job.verifyBuildRequest(logger)
+ self.assertEqual("", logger.getLogBuffer())
+
+ def test_verifyBuildRequest_archive_private_owners_mismatch(self):
+ archive = self.factory.makeArchive(private=True)
+ job = self.makeJob(archive=archive)
+ lfa = self.factory.makeLibraryFileAlias()
+ transaction.commit()
+ job.build.distro_arch_series.addOrUpdateChroot(lfa)
+ builder = MockBuilder()
+ job.setBuilder(builder, OkSlave())
+ logger = BufferLogger()
+ e = self.assertRaises(
+ SnapBuildArchiveOwnerMismatch, job.verifyBuildRequest, logger)
+ self.assertEqual(
+ "Snap package builds against private archives are only allowed "
+ "if the snap package owner and the archive owner are equal.",
+ str(e))
+
+ def test_verifyBuildRequest_no_chroot(self):
+ # verifyBuildRequest raises when the DAS has no chroot.
+ job = self.makeJob()
+ builder = MockBuilder()
+ job.setBuilder(builder, OkSlave())
+ logger = BufferLogger()
+ e = self.assertRaises(CannotBuild, job.verifyBuildRequest, logger)
+ self.assertIn("Missing chroot", str(e))
+
+ def test_extraBuildArgs_bzr(self):
+ # _extraBuildArgs returns appropriate arguments if asked to build a
+ # job for a Bazaar branch.
+ branch = self.factory.makeBranch()
+ job = self.makeJob(branch=branch)
+ expected_archives = get_sources_list_for_building(
+ job.build, job.build.distro_arch_series, None)
+ self.assertEqual({
+ "archive_private": False,
+ "archives": expected_archives,
+ "arch_tag": "i386",
+ "branch": branch.bzr_identity,
+ "name": u"test-snap",
+ }, job._extraBuildArgs())
+
+ def test_extraBuildArgs_git(self):
+ # _extraBuildArgs returns appropriate arguments if asked to build a
+ # job for a Git branch.
+ [ref] = self.factory.makeGitRefs()
+ job = self.makeJob(git_ref=ref)
+ expected_archives = get_sources_list_for_building(
+ job.build, job.build.distro_arch_series, None)
+ self.assertEqual({
+ "archive_private": False,
+ "archives": expected_archives,
+ "arch_tag": "i386",
+ "git_repository": ref.repository.git_https_url,
+ "git_path": ref.path,
+ "name": u"test-snap",
+ }, job._extraBuildArgs())
+
+ def test_composeBuildRequest(self):
+ job = self.makeJob()
+ lfa = self.factory.makeLibraryFileAlias(db_only=True)
+ job.build.distro_arch_series.addOrUpdateChroot(lfa)
+ self.assertEqual(
+ ('snap', job.build.distro_arch_series, {},
+ job._extraBuildArgs()),
+ job.composeBuildRequest(None))
+
+
+class MakeSnapBuildMixin:
+ """Provide the common makeBuild method returning a queued build."""
+
+ def makeBuild(self):
+ self.useFixture(FeatureFixture({SNAP_FEATURE_FLAG: u"on"}))
+ build = self.factory.makeSnapBuild(status=BuildStatus.BUILDING)
+ build.queueBuild()
+ return build
+
+ def makeUnmodifiableBuild(self):
+ self.useFixture(FeatureFixture({SNAP_FEATURE_FLAG: u"on"}))
+ build = self.factory.makeSnapBuild(status=BuildStatus.BUILDING)
+ build.distro_series.status = SeriesStatus.OBSOLETE
+ build.queueBuild()
+ return build
+
+
+class TestGetUploadMethodsForSnapBuild(
+ MakeSnapBuildMixin, TestGetUploadMethodsMixin, TestCaseWithFactory):
+ """IPackageBuild.getUpload-related methods work with Snap builds."""
+
+
+class TestVerifySuccessfulBuildForSnapBuild(
+ MakeSnapBuildMixin, TestVerifySuccessfulBuildMixin, TestCaseWithFactory):
+ """IBuildFarmJobBehaviour.verifySuccessfulBuild works."""
+
+
+class TestHandleStatusForSnapBuild(
+ MakeSnapBuildMixin, TestHandleStatusMixin, TrialTestCase,
+ fixtures.TestWithFixtures):
+ """IPackageBuild.handleStatus works with Snap builds."""
=== modified file 'lib/lp/soyuz/adapters/archivedependencies.py'
--- lib/lp/soyuz/adapters/archivedependencies.py 2014-12-11 21:24:19 +0000
+++ lib/lp/soyuz/adapters/archivedependencies.py 2015-08-03 14:55:59 +0000
@@ -133,7 +133,7 @@
def expand_dependencies(archive, distro_arch_series, pocket, component,
- source_package_name):
+ source_package_name, tools_archive=None):
"""Return the set of dependency archives, pockets and components.
:param archive: the context `IArchive`.
@@ -141,6 +141,8 @@
:param pocket: the context `PackagePublishingPocket`.
:param component: the context `IComponent`.
:param source_package_name: A source package name (as text)
+ :param tools_archive: if not None, an archive to use as an additional
+ dependency for build tools, just before the default primary archive.
:return: a list of (archive, distro_arch_series, pocket, [component]),
representing the dependencies defined by the given build context.
"""
@@ -172,6 +174,16 @@
(archive_dependency.dependency, distro_arch_series, pocket,
components))
+ # Consider build tools archive dependencies.
+ if tools_archive is not None:
+ components = [
+ component.name
+ for component in tools_archive.getComponentsForSeries(
+ distro_series)]
+ deps.append(
+ (tools_archive, distro_arch_series,
+ PackagePublishingPocket.RELEASE, components))
+
# Consider primary archive dependency override. Add the default
# primary archive dependencies if it's not present.
if archive.getArchiveDependency(
@@ -201,7 +213,8 @@
return deps
-def get_sources_list_for_building(build, distroarchseries, sourcepackagename):
+def get_sources_list_for_building(build, distroarchseries, sourcepackagename,
+ tools_archive=None):
"""Return the sources_list entries required to build the given item.
The entries are returned in the order that is most useful;
@@ -213,11 +226,14 @@
:param build: a context `IBuild`.
:param distroarchseries: A `IDistroArchSeries`
:param sourcepackagename: A source package name (as text)
+ :param tools_archive: if not None, an archive to use as an additional
+ dependency for build tools, just before the default primary archive.
:return: a deb sources_list entries (lines).
"""
deps = expand_dependencies(
build.archive, distroarchseries, build.pocket,
- build.current_component, sourcepackagename)
+ build.current_component, sourcepackagename,
+ tools_archive=tools_archive)
sources_list_lines = \
_get_sources_list_for_dependencies(deps)
=== modified file 'lib/lp/soyuz/doc/archive-dependencies.txt'
--- lib/lp/soyuz/doc/archive-dependencies.txt 2014-12-11 21:52:15 +0000
+++ lib/lp/soyuz/doc/archive-dependencies.txt 2015-08-03 14:55:59 +0000
@@ -499,6 +499,34 @@
deb http://archive.launchpad.dev/ubuntu hoary-updates
main restricted universe multiverse
+ >>> cprov.archive.external_dependencies = None
+
+
+== Tools archives ==
+
+We can force a "tools archive" to be added to the sources.list, which is
+useful for specialised build types.
+
+ >>> tools_archive = factory.makeArchive(
+ ... distribution=ubuntu, owner=cprov, name="tools",
+ ... purpose=ArchivePurpose.PPA)
+ >>> pub_binaries = test_publisher.getPubBinaries(
+ ... binaryname="tool", archive=tools_archive,
+ ... status=PackagePublishingStatus.PUBLISHED)
+ >>> for line in get_sources_list_for_building(
+ ... a_build, a_build.distro_arch_series,
+ ... a_build.source_package_release.name,
+ ... tools_archive=tools_archive):
+ ... print line
+ deb http://ppa.launchpad.dev/cprov/ppa/ubuntu hoary main
+ deb http://ppa.launchpad.dev/cprov/tools/ubuntu hoary main
+ deb http://archive.launchpad.dev/ubuntu hoary
+ main restricted universe multiverse
+ deb http://archive.launchpad.dev/ubuntu hoary-security
+ main restricted universe multiverse
+ deb http://archive.launchpad.dev/ubuntu hoary-updates
+ main restricted universe multiverse
+
== Overlays ==
=== modified file 'lib/lp/soyuz/interfaces/archive.py'
--- lib/lp/soyuz/interfaces/archive.py 2015-05-18 06:36:51 +0000
+++ lib/lp/soyuz/interfaces/archive.py 2015-08-03 14:55:59 +0000
@@ -2290,6 +2290,8 @@
:param ext_deps: The dependencies form field to check.
"""
errors = []
+ if ext_deps is None:
+ return errors
# The field can consist of multiple entries separated by
# newlines, so process each in turn.
for dep in ext_deps.splitlines():
Follow ups
