launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #22438
[Merge] lp:~cjwatson/launchpad/build-fast-cleanup into lp:launchpad
Colin Watson has proposed merging lp:~cjwatson/launchpad/build-fast-cleanup into lp:launchpad.
Commit message:
Send fast_cleanup: True to virtualised builds, since they can safely skip the final cleanup steps.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/build-fast-cleanup/+merge/344958
In the process, I refactored the various common bits of building the extra build arguments dict into BuildFarmJobBehaviourBase, so that I only had to calculate fast_cleanup in one place.
https://code.launchpad.net/~cjwatson/launchpad-buildd/faster-cleanup/+merge/344938 must be in place for this to be useful, but they can harmlessly be deployed in either order.
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/build-fast-cleanup into lp:launchpad.
=== modified file 'lib/lp/buildmaster/interfaces/buildfarmjobbehaviour.py'
--- lib/lp/buildmaster/interfaces/buildfarmjobbehaviour.py 2016-03-31 14:58:46 +0000
+++ lib/lp/buildmaster/interfaces/buildfarmjobbehaviour.py 2018-05-02 13:37:24 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2014 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Interface for build farm job behaviours."""
@@ -9,14 +9,41 @@
'IBuildFarmJobBehaviour',
]
-from zope.interface import Interface
+from zope.interface import (
+ Attribute,
+ Interface,
+ )
class IBuildFarmJobBehaviour(Interface):
+ builder_type = Attribute(
+ "The name of the builder type to use for this build, corresponding "
+ "to a launchpad-buildd build manager tag.")
+
+ distro_arch_series = Attribute("The `DistroArchSeries` to build against.")
+
def setBuilder(builder, slave):
"""Sets the associated builder and slave for this instance."""
+ def determineFilesToSend():
+ """Work out which files to send to the builder.
+
+ :return: A dict mapping filenames to dicts as follows::
+ 'sha1': SHA-1 of file content
+ 'url': URL from which the builder can fetch content
+ 'username' (optional): username to authenticate as
+ 'password' (optional): password to authenticate with
+ """
+
+ def extraBuildArgs(logger=None):
+ """Return extra arguments required by the builder for this build.
+
+ :param logger: An optional logger.
+ :return: A dict of builder arguments, or a Deferred resulting in the
+ same.
+ """
+
def composeBuildRequest(logger):
"""Compose parameters for a slave build request.
=== modified file 'lib/lp/buildmaster/model/buildfarmjobbehaviour.py'
--- lib/lp/buildmaster/model/buildfarmjobbehaviour.py 2016-10-12 14:02:19 +0000
+++ lib/lp/buildmaster/model/buildfarmjobbehaviour.py 2018-05-02 13:37:24 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Base and idle BuildFarmJobBehaviour classes."""
@@ -32,6 +32,7 @@
from lp.services.librarian.interfaces import ILibraryFileAliasSet
from lp.services.librarian.utils import copy_and_close
from lp.services.utils import sanitise_urls
+from lp.services.webapp import canonical_url
SLAVE_LOG_FILENAME = 'buildlog'
@@ -48,11 +49,39 @@
self.build = build
self._builder = None
+ @property
+ def distro_arch_series(self):
+ if self.build is not None:
+ return self.build.distro_arch_series
+ else:
+ return None
+
def setBuilder(self, builder, slave):
"""The builder should be set once and not changed."""
self._builder = builder
self._slave = slave
+ def determineFilesToSend(self):
+ """The default behaviour is to send no files."""
+ return {}
+
+ def extraBuildArgs(self, logger=None):
+ """The default behaviour is to send only common extra arguments."""
+ args = {}
+ args["arch_tag"] = self.distro_arch_series.architecturetag
+ args["archive_private"] = self.build.archive.private
+ args["build_url"] = canonical_url(self.build)
+ args["fast_cleanup"] = self._builder.virtualized
+ args["series"] = self.distro_arch_series.distroseries.name
+ return args
+
+ @defer.inlineCallbacks
+ def composeBuildRequest(self, logger):
+ args = yield self.extraBuildArgs(logger=logger)
+ defer.returnValue(
+ (self.builder_type, self.distro_arch_series,
+ self.determineFilesToSend(), args))
+
def verifyBuildRequest(self, logger):
"""The default behaviour is a no-op."""
pass
=== modified file 'lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py'
--- lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py 2018-02-14 11:13:47 +0000
+++ lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py 2018-05-02 13:37:24 +0000
@@ -116,6 +116,20 @@
'%s-%s' % (now.strftime("%Y%m%d-%H%M%S"), build_cookie),
upload_leaf)
+ def test_extraBuildArgs_virtualized(self):
+ # If the builder is virtualized, extraBuildArgs sends
+ # fast_cleanup: True.
+ behaviour = self._makeBehaviour(self._makeBuild())
+ behaviour.setBuilder(self.factory.makeBuilder(virtualized=True), None)
+ self.assertIs(True, behaviour.extraBuildArgs()["fast_cleanup"])
+
+ def test_extraBuildArgs_non_virtualized(self):
+ # If the builder is non-virtualized, extraBuildArgs sends
+ # fast_cleanup: False.
+ behaviour = self._makeBehaviour(self._makeBuild())
+ behaviour.setBuilder(self.factory.makeBuilder(virtualized=False), None)
+ self.assertIs(False, behaviour.extraBuildArgs()["fast_cleanup"])
+
class TestDispatchBuildToSlave(TestCase):
=== modified file 'lib/lp/code/model/recipebuilder.py'
--- lib/lp/code/model/recipebuilder.py 2018-03-01 17:36:31 +0000
+++ lib/lp/code/model/recipebuilder.py 2018-05-02 13:37:24 +0000
@@ -1,4 +1,4 @@
-# Copyright 2010-2017 Canonical Ltd. This software is licensed under the
+# Copyright 2010-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Code to build recipes on the buildfarm."""
@@ -24,7 +24,7 @@
ISourcePackageRecipeBuild,
)
from lp.services.config import config
-from lp.services.webapp import canonical_url
+from lp.services.propertycache import cachedproperty
from lp.soyuz.adapters.archivedependencies import (
get_primary_current_component,
get_sources_list_for_building,
@@ -36,22 +36,37 @@
class RecipeBuildBehaviour(BuildFarmJobBehaviourBase):
"""How to build a recipe on the build farm."""
+ builder_type = "sourcepackagerecipe"
+
# The list of build status values for which email notifications are
# allowed to be sent. It is up to each callback as to whether it will
# consider sending a notification but it won't do so if the status is not
# in this list.
ALLOWED_STATUS_NOTIFICATIONS = ['PACKAGEFAIL', 'DEPFAIL', 'CHROOTFAIL']
+ @cachedproperty
+ def distro_arch_series(self):
+ if self.build is not None and self._builder is not None:
+ return self.build.distroseries.getDistroArchSeriesByProcessor(
+ self._builder.processor)
+ else:
+ return None
+
@defer.inlineCallbacks
- def _extraBuildArgs(self, distroarchseries, logger=None):
+ def extraBuildArgs(self, logger=None):
"""
Return the extra arguments required by the slave for the given build.
"""
+ if self.distro_arch_series is None:
+ raise CannotBuild(
+ "Unable to find distroarchseries for %s in %s" %
+ (self._builder.processor.name,
+ self.build.distroseries.displayname))
+
# Build extra arguments.
- args = {}
- args['series'] = self.build.distroseries.name
+ args = yield super(RecipeBuildBehaviour, self).extraBuildArgs(
+ logger=logger)
args['suite'] = self.build.distroseries.getSuite(self.build.pocket)
- args['arch_tag'] = distroarchseries.architecturetag
requester = self.build.requester
if requester.preferredemail is None:
# Use a constant, known, name and email.
@@ -71,11 +86,9 @@
None).name
args['archives'], args['trusted_keys'] = (
yield get_sources_list_for_building(
- self.build, distroarchseries, None,
+ self.build, self.distro_arch_series, None,
tools_source=config.builddmaster.bzr_builder_sources_list,
logger=logger))
- args['archive_private'] = self.build.archive.private
- args['build_url'] = canonical_url(self.build)
# XXX cjwatson 2017-07-26: This duplicates "series", which is common
# to all build types; this name for it is deprecated and should be
# removed once launchpad-buildd no longer requires it.
@@ -84,18 +97,6 @@
args['git'] = True
defer.returnValue(args)
- @defer.inlineCallbacks
- def composeBuildRequest(self, logger):
- das = self.build.distroseries.getDistroArchSeriesByProcessor(
- self._builder.processor)
- if das is None:
- raise CannotBuild(
- "Unable to find distroarchseries for %s in %s" %
- (self._builder.processor.name,
- self.build.distroseries.displayname))
- args = yield self._extraBuildArgs(das, logger=logger)
- defer.returnValue(("sourcepackagerecipe", das, {}, args))
-
def verifyBuildRequest(self, logger):
"""Assert some pre-build checks.
=== modified file 'lib/lp/code/model/tests/test_recipebuilder.py'
--- lib/lp/code/model/tests/test_recipebuilder.py 2018-03-01 17:36:31 +0000
+++ lib/lp/code/model/tests/test_recipebuilder.py 2018-05-02 13:37:24 +0000
@@ -1,4 +1,4 @@
-# Copyright 2010-2017 Canonical Ltd. This software is licensed under the
+# Copyright 2010-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Test RecipeBuildBehaviour."""
@@ -70,7 +70,7 @@
layer = LaunchpadZopelessLayer
def makeJob(self, recipe_registrant=None, recipe_owner=None,
- archive=None, git=False):
+ archive=None, git=False, with_builder=False):
"""Create a sample `ISourcePackageRecipeBuild`."""
spn = self.factory.makeSourcePackageName("apackage")
if archive is None:
@@ -105,6 +105,10 @@
sourcepackage=sourcepackage, archive=archive,
recipe=recipe, requester=recipe_owner, distroseries=distroseries)
job = IBuildFarmJobBehaviour(spb)
+ if with_builder:
+ builder = MockBuilder()
+ builder.processor = processor
+ job.setBuilder(builder, None)
return job
@@ -171,16 +175,16 @@
@defer.inlineCallbacks
def test_extraBuildArgs(self):
- # _extraBuildArgs will return a sane set of additional arguments
+ # extraBuildArgs will return a sane set of additional arguments.
self._setBuilderConfig()
- job = self.makeJob()
+ job = self.makeJob(with_builder=True)
distroarchseries = job.build.distroseries.architectures[0]
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(
job.build, distroarchseries, None))
expected_archives.insert(
0, "deb http://foo %s main" % job.build.distroseries.name)
- args = yield job._extraBuildArgs(distroarchseries)
+ args = yield job.extraBuildArgs()
self.assertEqual({
'arch_tag': 'i386',
'archive_private': False,
@@ -190,6 +194,7 @@
'author_name': 'Joe User',
'build_url': canonical_url(job.build),
'distroseries_name': job.build.distroseries.name,
+ 'fast_cleanup': True,
'ogrecomponent': 'universe',
'recipe_text':
'# bzr-builder format 0.3 '
@@ -207,11 +212,9 @@
# build logs.
self._setBuilderConfig()
archive = self.factory.makeArchive(private=True)
- job = self.makeJob(archive=archive)
- distroarchseries = job.build.distroseries.architectures[0]
- extra_args = yield job._extraBuildArgs(distroarchseries)
- self.assertEqual(
- True, extra_args['archive_private'])
+ job = self.makeJob(archive=archive, with_builder=True)
+ extra_args = yield job.extraBuildArgs()
+ self.assertEqual(True, extra_args['archive_private'])
@defer.inlineCallbacks
def test_extraBuildArgs_team_owner_no_email(self):
@@ -224,9 +227,8 @@
recipe_owner = self.factory.makeTeam(
name='vikings', members=[recipe_registrant])
- job = self.makeJob(recipe_registrant, recipe_owner)
- distroarchseries = job.build.distroseries.architectures[0]
- extra_args = yield job._extraBuildArgs(distroarchseries)
+ job = self.makeJob(recipe_registrant, recipe_owner, with_builder=True)
+ extra_args = yield job.extraBuildArgs()
self.assertEqual(
"Launchpad Package Builder", extra_args['author_name'])
self.assertEqual("noreply@xxxxxxxxxxxxx", extra_args['author_email'])
@@ -241,9 +243,8 @@
name='vikings', email='everyone@xxxxxxxxxxxx',
members=[recipe_registrant])
- job = self.makeJob(recipe_registrant, recipe_owner)
- distroarchseries = job.build.distroseries.architectures[0]
- extra_args = yield job._extraBuildArgs(distroarchseries)
+ job = self.makeJob(recipe_registrant, recipe_owner, with_builder=True)
+ extra_args = yield job.extraBuildArgs()
self.assertEqual("Vikings", extra_args['author_name'])
self.assertEqual("everyone@xxxxxxxxxxxx", extra_args['author_email'])
@@ -254,28 +255,27 @@
owner = self.factory.makePerson()
with person_logged_in(owner):
owner.deactivate(comment='deactivating')
- job = self.makeJob(owner)
- distroarchseries = job.build.distroseries.architectures[0]
- extra_args = yield job._extraBuildArgs(distroarchseries)
+ job = self.makeJob(owner, with_builder=True)
+ extra_args = yield job.extraBuildArgs()
self.assertEqual(
"Launchpad Package Builder", extra_args['author_name'])
self.assertEqual("noreply@xxxxxxxxxxxxx", extra_args['author_email'])
@defer.inlineCallbacks
def test_extraBuildArgs_withBadConfigForBzrBuilderPPA(self):
- # Ensure _extraBuildArgs doesn't blow up with a badly formatted
+ # Ensure extraBuildArgs doesn't blow up with a badly formatted
# bzr_builder_sources_list in the config.
self.pushConfig(
"builddmaster",
bzr_builder_sources_list="deb http://foo %(series) main")
# (note the missing 's' in %(series)
- job = self.makeJob()
+ job = self.makeJob(with_builder=True)
distroarchseries = job.build.distroseries.architectures[0]
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(
job.build, distroarchseries, None))
logger = BufferLogger()
- extra_args = yield job._extraBuildArgs(distroarchseries, logger)
+ extra_args = yield job.extraBuildArgs(logger)
self.assertEqual({
'arch_tag': 'i386',
'archive_private': False,
@@ -285,6 +285,7 @@
'author_name': 'Joe User',
'build_url': canonical_url(job.build),
'distroseries_name': job.build.distroseries.name,
+ 'fast_cleanup': True,
'ogrecomponent': 'universe',
'recipe_text':
'# bzr-builder format 0.3 '
@@ -299,12 +300,12 @@
logger.getLogBuffer())
@defer.inlineCallbacks
- def test_extraBuildArgs_withNoBZrBuilderConfigSet(self):
- # Ensure _extraBuildArgs doesn't blow up when
+ def test_extraBuildArgs_withNoBzrBuilderConfigSet(self):
+ # Ensure extraBuildArgs doesn't blow up when
# bzr_builder_sources_list isn't set.
- job = self.makeJob()
+ job = self.makeJob(with_builder=True)
distroarchseries = job.build.distroseries.architectures[0]
- args = yield job._extraBuildArgs(distroarchseries)
+ args = yield job.extraBuildArgs()
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(
job.build, distroarchseries, None))
@@ -313,12 +314,12 @@
@defer.inlineCallbacks
def test_extraBuildArgs_git(self):
- job = self.makeJob(git=True)
+ job = self.makeJob(git=True, with_builder=True)
distroarchseries = job.build.distroseries.architectures[0]
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(
job.build, distroarchseries, None))
- extra_args = yield job._extraBuildArgs(distroarchseries)
+ extra_args = yield job.extraBuildArgs()
self.assertEqual({
'arch_tag': 'i386',
'archive_private': False,
@@ -328,6 +329,7 @@
'author_name': 'Joe User',
'build_url': canonical_url(job.build),
'distroseries_name': job.build.distroseries.name,
+ 'fast_cleanup': True,
'git': True,
'ogrecomponent': 'universe',
'recipe_text':
@@ -341,33 +343,30 @@
@defer.inlineCallbacks
def test_extraBuildArgs_archive_trusted_keys(self):
- # If the archive has a signing key, _extraBuildArgs sends it.
+ # If the archive has a signing key, extraBuildArgs sends it.
yield self.useFixture(InProcessKeyServerFixture()).start()
archive = self.factory.makeArchive()
key_path = os.path.join(gpgkeysdir, "ppa-sample@xxxxxxxxxxxxxxxxx")
yield IArchiveSigningKey(archive).setSigningKey(
key_path, async_keyserver=True)
- job = self.makeJob(archive=archive)
+ job = self.makeJob(archive=archive, with_builder=True)
distroarchseries = job.build.distroseries.architectures[0]
self.factory.makeBinaryPackagePublishingHistory(
distroarchseries=distroarchseries, pocket=job.build.pocket,
archive=archive, status=PackagePublishingStatus.PUBLISHED)
- args = yield job._extraBuildArgs(distroarchseries)
+ args = yield job.extraBuildArgs()
self.assertThat(args["trusted_keys"], MatchesListwise([
Base64KeyMatches("0D57E99656BEFB0897606EE9A022DD1F5001B46D"),
]))
@defer.inlineCallbacks
def test_composeBuildRequest(self):
- job = self.makeJob()
+ job = self.makeJob(with_builder=True)
test_publisher = SoyuzTestPublisher()
test_publisher.addFakeChroots(job.build.distroseries)
das = job.build.distroseries.nominatedarchindep
- builder = MockBuilder("bob-de-bouwer")
- builder.processor = das.processor
- job.setBuilder(builder, None)
build_request = yield job.composeBuildRequest(None)
- extra_args = yield job._extraBuildArgs(das)
+ extra_args = yield job.extraBuildArgs()
self.assertEqual(
('sourcepackagerecipe', das, {}, extra_args), build_request)
=== modified file 'lib/lp/snappy/model/snapbuildbehaviour.py'
--- lib/lp/snappy/model/snapbuildbehaviour.py 2018-04-23 11:14:38 +0000
+++ lib/lp/snappy/model/snapbuildbehaviour.py 2018-05-02 13:37:24 +0000
@@ -30,7 +30,6 @@
)
from lp.registry.interfaces.series import SeriesStatus
from lp.services.config import config
-from lp.services.webapp import canonical_url
from lp.snappy.interfaces.snap import SnapBuildArchiveOwnerMismatch
from lp.snappy.interfaces.snapbuild import ISnapBuild
from lp.soyuz.adapters.archivedependencies import (
@@ -44,6 +43,8 @@
class SnapBuildBehaviour(BuildFarmJobBehaviourBase):
"""Dispatches `SnapBuild` jobs to slaves."""
+ builder_type = "snap"
+
def getLogFileName(self):
das = self.build.distro_arch_series
@@ -79,12 +80,13 @@
"Missing chroot for %s" % build.distro_arch_series.displayname)
@defer.inlineCallbacks
- def _extraBuildArgs(self, logger=None):
+ def extraBuildArgs(self, logger=None):
"""
Return the extra arguments required by the slave for the given build.
"""
build = self.build
- args = {}
+ args = yield super(SnapBuildBehaviour, self).extraBuildArgs(
+ logger=logger)
if config.snappy.builder_proxy_host and build.snap.allow_internet:
token = yield self._requestProxyToken()
args["proxy_url"] = (
@@ -98,8 +100,6 @@
endpoint=config.snappy.builder_proxy_auth_api_endpoint,
token=token['username']))
args["name"] = build.snap.store_name or build.snap.name
- args["series"] = build.distro_series.name
- args["arch_tag"] = build.distro_arch_series.architecturetag
# XXX cjwatson 2015-08-03: Allow tools_source to be overridden at
# some more fine-grained level.
args["archives"], args["trusted_keys"] = (
@@ -108,8 +108,6 @@
tools_source=config.snappy.tools_source,
tools_fingerprint=config.snappy.tools_fingerprint,
logger=logger))
- args["archive_private"] = build.archive.private
- args["build_url"] = canonical_url(build)
if build.channels is not None:
# We have to remove the security proxy that Zope applies to this
# dict, since otherwise we'll be unable to serialise it to
@@ -168,11 +166,6 @@
token = json.loads(result)
defer.returnValue(token)
- @defer.inlineCallbacks
- def composeBuildRequest(self, logger):
- args = yield self._extraBuildArgs(logger=logger)
- defer.returnValue(("snap", self.build.distro_arch_series, {}, args))
-
def verifySuccessfulBuild(self):
"""See `IBuildFarmJobBehaviour`."""
# The implementation in BuildFarmJobBehaviourBase checks whether the
=== modified file 'lib/lp/snappy/tests/test_snapbuildbehaviour.py'
--- lib/lp/snappy/tests/test_snapbuildbehaviour.py 2018-04-23 11:14:38 +0000
+++ lib/lp/snappy/tests/test_snapbuildbehaviour.py 2018-05-02 13:37:24 +0000
@@ -80,7 +80,7 @@
self.pushConfig("snappy", tools_source=None, tools_fingerprint=None)
def makeJob(self, archive=None, pocket=PackagePublishingPocket.UPDATES,
- **kwargs):
+ with_builder=False, **kwargs):
"""Create a sample `ISnapBuildBehaviour`."""
if archive is None:
distribution = self.factory.makeDistribution(name="distro")
@@ -95,7 +95,12 @@
build = self.factory.makeSnapBuild(
archive=archive, distroarchseries=distroarchseries, pocket=pocket,
name="test-snap", **kwargs)
- return IBuildFarmJobBehaviour(build)
+ job = IBuildFarmJobBehaviour(build)
+ if with_builder:
+ builder = MockBuilder()
+ builder.processor = processor
+ job.setBuilder(builder, None)
+ return job
class TestSnapBuildBehaviour(TestSnapBuildBehaviourBase):
@@ -222,7 +227,7 @@
@defer.inlineCallbacks
def test_composeBuildRequest(self):
- job = self.makeJob()
+ job = self.makeJob(with_builder=True)
lfa = self.factory.makeLibraryFileAlias(db_only=True)
job.build.distro_arch_series.addOrUpdateChroot(lfa)
build_request = yield job.composeBuildRequest(None)
@@ -233,17 +238,17 @@
def test_requestProxyToken_unconfigured(self):
self.pushConfig("snappy", builder_proxy_auth_api_admin_secret=None)
branch = self.factory.makeBranch()
- job = self.makeJob(branch=branch)
+ job = self.makeJob(branch=branch, with_builder=True)
expected_exception_msg = (
"builder_proxy_auth_api_admin_secret is not configured.")
with ExpectedException(CannotBuild, expected_exception_msg):
- yield job._extraBuildArgs()
+ yield job.extraBuildArgs()
@defer.inlineCallbacks
def test_requestProxyToken(self):
branch = self.factory.makeBranch()
- job = self.makeJob(branch=branch)
- yield job._extraBuildArgs()
+ job = self.makeJob(branch=branch, with_builder=True)
+ yield job.extraBuildArgs()
self.assertThat(self.mock_proxy_api.calls, MatchesListwise([
MatchesListwise([
MatchesListwise([
@@ -265,14 +270,14 @@
@defer.inlineCallbacks
def test_extraBuildArgs_bzr(self):
- # _extraBuildArgs returns appropriate arguments if asked to build a
+ # extraBuildArgs returns appropriate arguments if asked to build a
# job for a Bazaar branch.
branch = self.factory.makeBranch()
- job = self.makeJob(branch=branch)
+ job = self.makeJob(branch=branch, with_builder=True)
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(
job.build, job.build.distro_arch_series, None))
- args = yield job._extraBuildArgs()
+ args = yield job.extraBuildArgs()
self.assertEqual({
"archive_private": False,
"archives": expected_archives,
@@ -280,6 +285,7 @@
"branch": branch.bzr_identity,
"build_source_tarball": False,
"build_url": canonical_url(job.build),
+ "fast_cleanup": True,
"name": "test-snap",
"proxy_url": self.proxy_url,
"revocation_endpoint": self.revocation_endpoint,
@@ -289,20 +295,21 @@
@defer.inlineCallbacks
def test_extraBuildArgs_git(self):
- # _extraBuildArgs returns appropriate arguments if asked to build a
+ # extraBuildArgs returns appropriate arguments if asked to build a
# job for a Git branch.
[ref] = self.factory.makeGitRefs()
- job = self.makeJob(git_ref=ref)
+ job = self.makeJob(git_ref=ref, with_builder=True)
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(
job.build, job.build.distro_arch_series, None))
- args = yield job._extraBuildArgs()
+ args = yield job.extraBuildArgs()
self.assertEqual({
"archive_private": False,
"archives": expected_archives,
"arch_tag": "i386",
"build_source_tarball": False,
"build_url": canonical_url(job.build),
+ "fast_cleanup": True,
"git_repository": ref.repository.git_https_url,
"git_path": ref.name,
"name": "test-snap",
@@ -314,21 +321,23 @@
@defer.inlineCallbacks
def test_extraBuildArgs_git_HEAD(self):
- # _extraBuildArgs returns appropriate arguments if asked to build a
+ # extraBuildArgs returns appropriate arguments if asked to build a
# job for the default branch in a Launchpad-hosted Git repository.
[ref] = self.factory.makeGitRefs()
removeSecurityProxy(ref.repository)._default_branch = ref.path
- job = self.makeJob(git_ref=ref.repository.getRefByPath("HEAD"))
+ job = self.makeJob(
+ git_ref=ref.repository.getRefByPath("HEAD"), with_builder=True)
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(
job.build, job.build.distro_arch_series, None))
- args = yield job._extraBuildArgs()
+ args = yield job.extraBuildArgs()
self.assertEqual({
"archive_private": False,
"archives": expected_archives,
"arch_tag": "i386",
"build_source_tarball": False,
"build_url": canonical_url(job.build),
+ "fast_cleanup": True,
"git_repository": ref.repository.git_https_url,
"name": "test-snap",
"proxy_url": self.proxy_url,
@@ -339,22 +348,23 @@
@defer.inlineCallbacks
def test_extraBuildArgs_git_url(self):
- # _extraBuildArgs returns appropriate arguments if asked to build a
+ # extraBuildArgs returns appropriate arguments if asked to build a
# job for a Git branch backed by a URL for an external repository.
url = "https://git.example.org/foo";
ref = self.factory.makeGitRefRemote(
repository_url=url, path="refs/heads/master")
- job = self.makeJob(git_ref=ref)
+ job = self.makeJob(git_ref=ref, with_builder=True)
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(
job.build, job.build.distro_arch_series, None))
- args = yield job._extraBuildArgs()
+ args = yield job.extraBuildArgs()
self.assertEqual({
"archive_private": False,
"archives": expected_archives,
"arch_tag": "i386",
"build_source_tarball": False,
"build_url": canonical_url(job.build),
+ "fast_cleanup": True,
"git_repository": url,
"git_path": "master",
"name": "test-snap",
@@ -366,21 +376,22 @@
@defer.inlineCallbacks
def test_extraBuildArgs_git_url_HEAD(self):
- # _extraBuildArgs returns appropriate arguments if asked to build a
+ # extraBuildArgs returns appropriate arguments if asked to build a
# job for the default branch in an external Git repository.
url = "https://git.example.org/foo";
ref = self.factory.makeGitRefRemote(repository_url=url, path="HEAD")
- job = self.makeJob(git_ref=ref)
+ job = self.makeJob(git_ref=ref, with_builder=True)
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(
job.build, job.build.distro_arch_series, None))
- args = yield job._extraBuildArgs()
+ args = yield job.extraBuildArgs()
self.assertEqual({
"archive_private": False,
"archives": expected_archives,
"arch_tag": "i386",
"build_source_tarball": False,
"build_url": canonical_url(job.build),
+ "fast_cleanup": True,
"git_repository": url,
"name": "test-snap",
"proxy_url": self.proxy_url,
@@ -391,62 +402,61 @@
@defer.inlineCallbacks
def test_extraBuildArgs_prefers_store_name(self):
- # For the "name" argument, _extraBuildArgs prefers Snap.store_name
+ # For the "name" argument, extraBuildArgs prefers Snap.store_name
# over Snap.name if the former is set.
- job = self.makeJob(store_name="something-else")
- args = yield job._extraBuildArgs()
+ job = self.makeJob(store_name="something-else", with_builder=True)
+ args = yield job.extraBuildArgs()
self.assertEqual("something-else", args["name"])
@defer.inlineCallbacks
def test_extraBuildArgs_archive_trusted_keys(self):
- # If the archive has a signing key, _extraBuildArgs sends it.
+ # If the archive has a signing key, extraBuildArgs sends it.
yield self.useFixture(InProcessKeyServerFixture()).start()
archive = self.factory.makeArchive()
key_path = os.path.join(gpgkeysdir, "ppa-sample@xxxxxxxxxxxxxxxxx")
yield IArchiveSigningKey(archive).setSigningKey(
key_path, async_keyserver=True)
- job = self.makeJob(archive=archive)
+ job = self.makeJob(archive=archive, with_builder=True)
self.factory.makeBinaryPackagePublishingHistory(
distroarchseries=job.build.distro_arch_series,
pocket=job.build.pocket, archive=archive,
status=PackagePublishingStatus.PUBLISHED)
- args = yield job._extraBuildArgs()
+ args = yield job.extraBuildArgs()
self.assertThat(args["trusted_keys"], MatchesListwise([
Base64KeyMatches("0D57E99656BEFB0897606EE9A022DD1F5001B46D"),
]))
@defer.inlineCallbacks
def test_extraBuildArgs_channels(self):
- # If the build needs particular channels, _extraBuildArgs sends
- # them.
- job = self.makeJob(channels={"snapcraft": "edge"})
+ # If the build needs particular channels, extraBuildArgs sends them.
+ job = self.makeJob(channels={"snapcraft": "edge"}, with_builder=True)
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(
job.build, job.build.distro_arch_series, None))
- args = yield job._extraBuildArgs()
+ args = yield job.extraBuildArgs()
self.assertFalse(isProxy(args["channels"]))
self.assertEqual({"snapcraft": "edge"}, args["channels"])
@defer.inlineCallbacks
def test_extraBuildArgs_disallow_internet(self):
# If external network access is not allowed for the snap,
- # _extraBuildArgs does not dispatch a proxy token.
- job = self.makeJob(allow_internet=False)
- args = yield job._extraBuildArgs()
+ # extraBuildArgs does not dispatch a proxy token.
+ job = self.makeJob(allow_internet=False, with_builder=True)
+ args = yield job.extraBuildArgs()
self.assertNotIn("proxy_url", args)
self.assertNotIn("revocation_endpoint", args)
@defer.inlineCallbacks
def test_extraBuildArgs_build_source_tarball(self):
- # If the snap requests building of a source tarball, _extraBuildArgs
+ # If the snap requests building of a source tarball, extraBuildArgs
# sends the appropriate arguments.
- job = self.makeJob(build_source_tarball=True)
- args = yield job._extraBuildArgs()
+ job = self.makeJob(build_source_tarball=True, with_builder=True)
+ args = yield job.extraBuildArgs()
self.assertTrue(args["build_source_tarball"])
@defer.inlineCallbacks
def test_composeBuildRequest_proxy_url_set(self):
- job = self.makeJob()
+ job = self.makeJob(with_builder=True)
build_request = yield job.composeBuildRequest(None)
proxy_url = ("http://{username}:{password}";
"@{host}:{port}".format(
@@ -462,7 +472,8 @@
# composeBuildRequest raises CannotBuild.
branch = self.factory.makeBranch()
owner = self.factory.makePerson(name="snap-owner")
- job = self.makeJob(registrant=owner, owner=owner, branch=branch)
+ job = self.makeJob(
+ registrant=owner, owner=owner, branch=branch, with_builder=True)
branch.destroySelf(break_references=True)
self.assertIsNone(job.build.snap.branch)
self.assertIsNone(job.build.snap.git_repository)
@@ -478,7 +489,8 @@
repository = self.factory.makeGitRepository()
[ref] = self.factory.makeGitRefs(repository=repository)
owner = self.factory.makePerson(name="snap-owner")
- job = self.makeJob(registrant=owner, owner=owner, git_ref=ref)
+ job = self.makeJob(
+ registrant=owner, owner=owner, git_ref=ref, with_builder=True)
repository.removeRefs([ref.path])
self.assertIsNone(job.build.snap.git_ref)
expected_exception_msg = ("Source branch/repository for "
=== modified file 'lib/lp/soyuz/model/binarypackagebuildbehaviour.py'
--- lib/lp/soyuz/model/binarypackagebuildbehaviour.py 2018-03-01 17:36:31 +0000
+++ lib/lp/soyuz/model/binarypackagebuildbehaviour.py 2018-05-02 13:37:24 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Builder behaviour for binary package builds."""
@@ -20,10 +20,7 @@
BuildFarmJobBehaviourBase,
)
from lp.registry.interfaces.pocket import PackagePublishingPocket
-from lp.services.webapp import (
- canonical_url,
- urlappend,
- )
+from lp.services.webapp import urlappend
from lp.soyuz.adapters.archivedependencies import (
get_primary_current_component,
get_sources_list_for_building,
@@ -36,6 +33,8 @@
class BinaryPackageBuildBehaviour(BuildFarmJobBehaviourBase):
"""Define the behaviour of binary package builds."""
+ builder_type = "binarypackage"
+
def getLogFileName(self):
"""See `IBuildPackageJob`."""
sourcename = self.build.source_package_release.name
@@ -60,6 +59,7 @@
state))
def determineFilesToSend(self):
+ """See `IBuildFarmJobBehaviour`."""
# Build filemap structure with the files required in this build
# and send them to the slave.
if self.build.archive.private:
@@ -85,13 +85,6 @@
'password': self.build.archive.buildd_secret}
return filemap
- @defer.inlineCallbacks
- def composeBuildRequest(self, logger):
- args = yield self._extraBuildArgs(self.build, logger=logger)
- defer.returnValue(
- ("binarypackage", self.build.distro_arch_series,
- self.determineFilesToSend(), args))
-
def verifyBuildRequest(self, logger):
"""Assert some pre-build checks.
@@ -137,19 +130,19 @@
build.distro_series.name))
@defer.inlineCallbacks
- def _extraBuildArgs(self, build, logger=None):
+ def extraBuildArgs(self, logger=None):
"""
Return the extra arguments required by the slave for the given build.
"""
+ build = self.build
das = build.distro_arch_series
# Build extra arguments.
- args = {}
+ args = yield super(BinaryPackageBuildBehaviour, self).extraBuildArgs(
+ logger=logger)
args['arch_indep'] = build.arch_indep
args['distribution'] = das.distroseries.distribution.name
- args['series'] = das.distroseries.name
args['suite'] = das.distroseries.getSuite(build.pocket)
- args['arch_tag'] = das.architecturetag
archive_purpose = build.archive.purpose
if (archive_purpose == ArchivePurpose.PPA and
@@ -171,8 +164,6 @@
args['archives'], args['trusted_keys'] = (
yield get_sources_list_for_building(
build, das, build.source_package_release.name, logger=logger))
- args['archive_private'] = build.archive.private
- args['build_url'] = canonical_url(build)
args['build_debug_symbols'] = build.archive.build_debug_symbols
defer.returnValue(args)
=== modified file 'lib/lp/soyuz/model/livefsbuildbehaviour.py'
--- lib/lp/soyuz/model/livefsbuildbehaviour.py 2018-03-01 17:36:31 +0000
+++ lib/lp/soyuz/model/livefsbuildbehaviour.py 2018-05-02 13:37:24 +0000
@@ -1,4 +1,4 @@
-# Copyright 2014-2017 Canonical Ltd. This software is licensed under the
+# Copyright 2014-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""An `IBuildFarmJobBehaviour` for `LiveFSBuild`.
@@ -24,7 +24,6 @@
BuildFarmJobBehaviourBase,
)
from lp.registry.interfaces.series import SeriesStatus
-from lp.services.webapp import canonical_url
from lp.soyuz.adapters.archivedependencies import (
get_sources_list_for_building,
)
@@ -38,6 +37,8 @@
class LiveFSBuildBehaviour(BuildFarmJobBehaviourBase):
"""Dispatches `LiveFSBuild` jobs to slaves."""
+ builder_type = "livefs"
+
def getLogFileName(self):
das = self.build.distro_arch_series
archname = das.architecturetag
@@ -76,33 +77,26 @@
"Missing chroot for %s" % build.distro_arch_series.displayname)
@defer.inlineCallbacks
- def _extraBuildArgs(self, logger=None):
+ def extraBuildArgs(self, logger=None):
"""
Return the extra arguments required by the slave for the given build.
"""
build = self.build
+ args = yield super(LiveFSBuildBehaviour, self).extraBuildArgs(
+ logger=logger)
# Non-trivial metadata values may have been security-wrapped, which
# is pointless here and just gets in the way of xmlrpclib
# serialisation.
- args = dict(removeSecurityProxy(build.livefs.metadata))
+ args.update(removeSecurityProxy(build.livefs.metadata))
if build.metadata_override is not None:
args.update(removeSecurityProxy(build.metadata_override))
- args["series"] = build.distro_series.name
args["pocket"] = build.pocket.name.lower()
- args["arch_tag"] = build.distro_arch_series.architecturetag
args["datestamp"] = build.version
args["archives"], args["trusted_keys"] = (
yield get_sources_list_for_building(
build, build.distro_arch_series, None, logger=logger))
- args["archive_private"] = build.archive.private
- args["build_url"] = canonical_url(build)
defer.returnValue(args)
- @defer.inlineCallbacks
- def composeBuildRequest(self, logger):
- args = yield self._extraBuildArgs(logger=logger)
- defer.returnValue(("livefs", self.build.distro_arch_series, {}, args))
-
def verifySuccessfulBuild(self):
"""See `IBuildFarmJobBehaviour`."""
# The implementation in BuildFarmJobBehaviourBase checks whether the
=== modified file 'lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py'
--- lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py 2018-03-01 17:36:31 +0000
+++ lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py 2018-05-02 13:37:24 +0000
@@ -143,6 +143,7 @@
'build_debug_symbols': archive.build_debug_symbols,
'build_url': canonical_url(build),
'distribution': das.distroseries.distribution.name,
+ 'fast_cleanup': builder.virtualized,
'ogrecomponent': component,
'series': ds_name,
'suite': suite,
@@ -333,18 +334,24 @@
@defer.inlineCallbacks
def test_arch_indep(self):
# BinaryPackageBuild.arch_indep is passed through to the slave.
+ builder = self.factory.makeBuilder()
build = self.factory.makeBinaryPackageBuild(arch_indep=False)
- extra_args = yield IBuildFarmJobBehaviour(build)._extraBuildArgs(build)
+ behaviour = IBuildFarmJobBehaviour(build)
+ behaviour.setBuilder(builder, None)
+ extra_args = yield behaviour.extraBuildArgs()
self.assertFalse(extra_args['arch_indep'])
build = self.factory.makeBinaryPackageBuild(arch_indep=True)
- extra_args = yield IBuildFarmJobBehaviour(build)._extraBuildArgs(build)
+ behaviour = IBuildFarmJobBehaviour(build)
+ behaviour.setBuilder(builder, None)
+ extra_args = yield behaviour.extraBuildArgs()
self.assertTrue(extra_args['arch_indep'])
@defer.inlineCallbacks
def test_extraBuildArgs_archive_trusted_keys(self):
- # If the archive has a signing key, _extraBuildArgs sends it.
+ # If the archive has a signing key, extraBuildArgs sends it.
yield self.useFixture(InProcessKeyServerFixture()).start()
archive = self.factory.makeArchive()
+ builder = self.factory.makeBuilder()
key_path = os.path.join(gpgkeysdir, "ppa-sample@xxxxxxxxxxxxxxxxx")
yield IArchiveSigningKey(archive).setSigningKey(
key_path, async_keyserver=True)
@@ -352,7 +359,9 @@
self.factory.makeBinaryPackagePublishingHistory(
distroarchseries=build.distro_arch_series, pocket=build.pocket,
archive=archive, status=PackagePublishingStatus.PUBLISHED)
- args = yield IBuildFarmJobBehaviour(build)._extraBuildArgs(build)
+ behaviour = IBuildFarmJobBehaviour(build)
+ behaviour.setBuilder(builder, None)
+ args = yield behaviour.extraBuildArgs()
self.assertThat(args["trusted_keys"], MatchesListwise([
Base64KeyMatches("0D57E99656BEFB0897606EE9A022DD1F5001B46D"),
]))
=== modified file 'lib/lp/soyuz/tests/test_livefsbuildbehaviour.py'
--- lib/lp/soyuz/tests/test_livefsbuildbehaviour.py 2018-03-01 17:36:31 +0000
+++ lib/lp/soyuz/tests/test_livefsbuildbehaviour.py 2018-05-02 13:37:24 +0000
@@ -69,7 +69,7 @@
self.useFixture(FeatureFixture({LIVEFS_FEATURE_FLAG: "on"}))
def makeJob(self, archive=None, pocket=PackagePublishingPocket.RELEASE,
- **kwargs):
+ with_builder=False, **kwargs):
"""Create a sample `ILiveFSBuildBehaviour`."""
if archive is None:
distribution = self.factory.makeDistribution(name="distro")
@@ -84,7 +84,12 @@
build = self.factory.makeLiveFSBuild(
archive=archive, distroarchseries=distroarchseries, pocket=pocket,
name="test-livefs", **kwargs)
- return IBuildFarmJobBehaviour(build)
+ job = IBuildFarmJobBehaviour(build)
+ if with_builder:
+ builder = MockBuilder()
+ builder.processor = processor
+ job.setBuilder(builder, None)
+ return job
class TestLiveFSBuildBehaviour(TestLiveFSBuildBehaviourBase):
@@ -185,20 +190,22 @@
@defer.inlineCallbacks
def test_extraBuildArgs(self):
- # _extraBuildArgs returns a reasonable set of additional arguments.
+ # extraBuildArgs returns a reasonable set of additional arguments.
job = self.makeJob(
date_created=datetime(2014, 4, 25, 10, 38, 0, tzinfo=pytz.UTC),
- metadata={"project": "distro", "subproject": "special"})
+ metadata={"project": "distro", "subproject": "special"},
+ with_builder=True)
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(
job.build, job.build.distro_arch_series, None))
- extra_args = yield job._extraBuildArgs()
+ extra_args = yield job.extraBuildArgs()
self.assertEqual({
"archive_private": False,
"archives": expected_archives,
"arch_tag": "i386",
"build_url": canonical_url(job.build),
"datestamp": "20140425-103800",
+ "fast_cleanup": True,
"pocket": "release",
"project": "distro",
"subproject": "special",
@@ -208,50 +215,51 @@
@defer.inlineCallbacks
def test_extraBuildArgs_proposed(self):
- # _extraBuildArgs returns appropriate arguments if asked to build a
+ # extraBuildArgs returns appropriate arguments if asked to build a
# job for -proposed.
job = self.makeJob(
pocket=PackagePublishingPocket.PROPOSED,
- metadata={"project": "distro"})
- args = yield job._extraBuildArgs()
+ metadata={"project": "distro"}, with_builder=True)
+ args = yield job.extraBuildArgs()
self.assertEqual("unstable", args["series"])
self.assertEqual("proposed", args["pocket"])
@defer.inlineCallbacks
def test_extraBuildArgs_no_security_proxy(self):
- # _extraBuildArgs returns an object without security wrapping, even
+ # extraBuildArgs returns an object without security wrapping, even
# if values in the metadata are (say) lists and hence get proxied by
# Zope.
- job = self.makeJob(metadata={"lb_args": ["--option=value"]})
- args = yield job._extraBuildArgs()
+ job = self.makeJob(
+ metadata={"lb_args": ["--option=value"]}, with_builder=True)
+ args = yield job.extraBuildArgs()
self.assertEqual(["--option=value"], args["lb_args"])
self.assertIsNot(Proxy, type(args["lb_args"]))
@defer.inlineCallbacks
def test_extraBuildArgs_archive_trusted_keys(self):
- # If the archive has a signing key, _extraBuildArgs sends it.
+ # If the archive has a signing key, extraBuildArgs sends it.
yield self.useFixture(InProcessKeyServerFixture()).start()
archive = self.factory.makeArchive()
key_path = os.path.join(gpgkeysdir, "ppa-sample@xxxxxxxxxxxxxxxxx")
yield IArchiveSigningKey(archive).setSigningKey(
key_path, async_keyserver=True)
- job = self.makeJob(archive=archive)
+ job = self.makeJob(archive=archive, with_builder=True)
self.factory.makeBinaryPackagePublishingHistory(
distroarchseries=job.build.distro_arch_series,
pocket=job.build.pocket, archive=archive,
status=PackagePublishingStatus.PUBLISHED)
- args = yield job._extraBuildArgs()
+ args = yield job.extraBuildArgs()
self.assertThat(args["trusted_keys"], MatchesListwise([
Base64KeyMatches("0D57E99656BEFB0897606EE9A022DD1F5001B46D"),
]))
@defer.inlineCallbacks
def test_composeBuildRequest(self):
- job = self.makeJob()
+ job = self.makeJob(with_builder=True)
lfa = self.factory.makeLibraryFileAlias(db_only=True)
job.build.distro_arch_series.addOrUpdateChroot(lfa)
build_request = yield job.composeBuildRequest(None)
- args = yield job._extraBuildArgs()
+ args = yield job.extraBuildArgs()
self.assertEqual(
('livefs', job.build.distro_arch_series, {}, args), build_request)
Follow ups
