launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #22636
[Merge] lp:~cjwatson/launchpad/snap-daily-builds-request-builds into lp:launchpad
Colin Watson has proposed merging lp:~cjwatson/launchpad/snap-daily-builds-request-builds into lp:launchpad with lp:~cjwatson/launchpad/git-ref-remote-blob as a prerequisite.
Commit message:
Make automatic builds of snaps honour build-on architectures.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/snap-daily-builds-request-builds/+merge/348187
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/snap-daily-builds-request-builds into lp:launchpad.
=== modified file 'lib/lp/snappy/adapters/buildarch.py'
--- lib/lp/snappy/adapters/buildarch.py 2018-06-18 22:36:14 +0000
+++ lib/lp/snappy/adapters/buildarch.py 2018-06-18 22:36:14 +0000
@@ -122,12 +122,12 @@
def determine_architectures_to_build(snapcraft_data, supported_arches):
- """Return a set of architectures to build based on snapcraft.yaml.
+ """Return a list of architectures to build based on snapcraft.yaml.
:param snapcraft_data: A parsed snapcraft.yaml.
:param supported_arches: An ordered list of all architecture tags that
we can create builds for.
- :return: a set of `SnapBuildInstance`s.
+ :return: a list of `SnapBuildInstance`s.
"""
architectures_list = snapcraft_data.get("architectures")
@@ -162,10 +162,10 @@
if duplicates:
raise DuplicateBuildOnError(duplicates)
- architectures_to_build = set()
+ architectures_to_build = []
for arch in architectures:
try:
- architectures_to_build.add(
+ architectures_to_build.append(
SnapBuildInstance(arch, supported_arches))
except UnsupportedBuildOnError:
# Snaps are allowed to declare that they build on architectures
=== modified file 'lib/lp/snappy/interfaces/snap.py'
--- lib/lp/snappy/interfaces/snap.py 2018-06-18 22:36:14 +0000
+++ lib/lp/snappy/interfaces/snap.py 2018-06-18 22:36:14 +0000
@@ -406,6 +406,7 @@
"""
def requestBuildsFromJob(requester, archive, pocket, channels=None,
+ allow_failures=False, fetch_snapcraft_yaml=True,
logger=None):
"""Synchronous part of `Snap.requestBuilds`.
@@ -416,6 +417,13 @@
:param pocket: The pocket that should be targeted.
:param channels: A dictionary mapping snap names to channels to use
for these builds.
+ :param allow_failures: If True, log exceptions other than "already
+ pending" from individual build requests; if False, raise them to
+ the caller.
+ :param fetch_snapcraft_yaml: If True, fetch snapcraft.yaml from the
+ appropriate branch or repository and use it to decide which
+ builds to request; if False, fall back to building for all
+ supported architectures.
:param logger: An optional logger.
:return: A sequence of `ISnapBuild` instances.
"""
@@ -474,12 +482,17 @@
@operation_returns_collection_of(Interface)
@export_write_operation()
@operation_for_version("devel")
- def requestAutoBuilds(allow_failures=False, logger=None):
+ def requestAutoBuilds(allow_failures=False, fetch_snapcraft_yaml=False,
+ logger=None):
"""Create and return automatic builds for this snap package.
:param allow_failures: If True, log exceptions other than "already
pending" from individual build requests; if False, raise them to
the caller.
+ :param fetch_snapcraft_yaml: If True, fetch snapcraft.yaml from the
+ appropriate branch or repository and use it to decide which
+ builds to request; if False, fall back to building for all
+ supported architectures.
:param logger: An optional logger.
:raises CannotRequestAutoBuilds: if no auto_build_archive or
auto_build_pocket is set.
=== modified file 'lib/lp/snappy/model/snap.py'
--- lib/lp/snappy/model/snap.py 2018-06-18 22:36:14 +0000
+++ lib/lp/snappy/model/snap.py 2018-06-18 22:36:14 +0000
@@ -552,18 +552,22 @@
return self.getBuildRequest(job.job_id)
def requestBuildsFromJob(self, requester, archive, pocket, channels=None,
+ allow_failures=False, fetch_snapcraft_yaml=True,
logger=None):
"""See `ISnap`."""
- try:
- snapcraft_data = removeSecurityProxy(
- getUtility(ISnapSet).getSnapcraftYaml(self))
- except CannotFetchSnapcraftYaml as e:
- if not e.unsupported_remote:
- raise
- # The only reason we can't fetch the file is because we don't
- # support fetching from this repository's host. In this case
- # the best thing is to fall back to building for all supported
- # architectures.
+ if fetch_snapcraft_yaml:
+ try:
+ snapcraft_data = removeSecurityProxy(
+ getUtility(ISnapSet).getSnapcraftYaml(self))
+ except CannotFetchSnapcraftYaml as e:
+ if not e.unsupported_remote:
+ raise
+ # The only reason we can't fetch the file is because we
+ # don't support fetching from this repository's host. In
+ # this case the best thing is to fall back to building for
+ # all supported architectures.
+ snapcraft_data = {}
+ else:
snapcraft_data = {}
# Sort by Processor.id for determinism. This is chosen to be the
# same order as in BinaryPackageBuildSet.createForSource, to
@@ -592,11 +596,18 @@
logger.warning(
" - %s/%s/%s: %s",
self.owner.name, self.name, arch, e)
+ except Exception as e:
+ if not allow_failures:
+ raise
+ elif logger is not None:
+ logger.exception(
+ " - %s/%s/%s: %s",
+ self.owner.name, self.name, arch, e)
return builds
- def requestAutoBuilds(self, allow_failures=False, logger=None):
+ def requestAutoBuilds(self, allow_failures=False,
+ fetch_snapcraft_yaml=False, logger=None):
"""See `ISnap`."""
- builds = []
if self.auto_build_archive is None:
raise CannotRequestAutoBuilds("auto_build_archive")
if self.auto_build_pocket is None:
@@ -606,29 +617,10 @@
logger.debug(
"Scheduling builds of snap package %s/%s",
self.owner.name, self.name)
- for arch in self.getAllowedArchitectures():
- try:
- build = self.requestBuild(
- self.owner, self.auto_build_archive, arch,
- self.auto_build_pocket, self.auto_build_channels)
- if logger is not None:
- logger.debug(
- " - %s/%s/%s: Build requested.",
- self.owner.name, self.name, arch.architecturetag)
- builds.append(build)
- except SnapBuildAlreadyPending as e:
- if logger is not None:
- logger.warning(
- " - %s/%s/%s: %s",
- self.owner.name, self.name, arch.architecturetag, e)
- except Exception as e:
- if not allow_failures:
- raise
- elif logger is not None:
- logger.exception(
- " - %s/%s/%s: %s",
- self.owner.name, self.name, arch.architecturetag, e)
- return builds
+ return self.requestBuildsFromJob(
+ self.owner, self.auto_build_archive, self.auto_build_pocket,
+ channels=self.auto_build_channels, allow_failures=allow_failures,
+ fetch_snapcraft_yaml=fetch_snapcraft_yaml, logger=logger)
def getBuildRequest(self, job_id):
"""See `ISnap`."""
@@ -1116,7 +1108,7 @@
builds = []
for snap in snaps:
builds.extend(snap.requestAutoBuilds(
- allow_failures=True, logger=logger))
+ allow_failures=True, fetch_snapcraft_yaml=True, logger=logger))
return builds
def detachFromBranch(self, branch):
=== modified file 'lib/lp/snappy/tests/test_snap.py'
--- lib/lp/snappy/tests/test_snap.py 2018-06-18 22:36:14 +0000
+++ lib/lp/snappy/tests/test_snap.py 2018-06-18 22:36:14 +0000
@@ -1285,9 +1285,10 @@
def makeAutoBuildableSnap(self, **kwargs):
processor = self.factory.makeProcessor(supports_virtualized=True)
das = self.makeBuildableDistroArchSeries(processor=processor)
+ [git_ref] = self.factory.makeGitRefs()
snap = self.factory.makeSnap(
distroseries=das.distroseries, processors=[das.processor],
- auto_build=True, **kwargs)
+ git_ref=git_ref, auto_build=True, **kwargs)
return das, snap
def test_makeAutoBuilds(self):
@@ -1296,7 +1297,11 @@
self.assertEqual([], getUtility(ISnapSet).makeAutoBuilds())
das, snap = self.makeAutoBuildableSnap(is_stale=True)
logger = BufferLogger()
- [build] = getUtility(ISnapSet).makeAutoBuilds(logger=logger)
+ with GitHostingFixture(blob=dedent("""\
+ architectures:
+ - build-on: %s
+ """) % das.architecturetag):
+ [build] = getUtility(ISnapSet).makeAutoBuilds(logger=logger)
self.assertThat(build, MatchesStructure.byEquality(
requester=snap.owner, snap=snap, distro_arch_series=das,
status=BuildStatus.NEEDSBUILD,
@@ -1319,7 +1324,11 @@
requester=snap.owner, snap=snap, archive=snap.auto_build_archive,
distroarchseries=das)
logger = BufferLogger()
- builds = getUtility(ISnapSet).makeAutoBuilds(logger=logger)
+ with GitHostingFixture(blob=dedent("""\
+ architectures:
+ - build-on: %s
+ """) % das.architecturetag):
+ builds = getUtility(ISnapSet).makeAutoBuilds(logger=logger)
self.assertEqual([], builds)
self.assertEqual([], logger.getLogBuffer().splitlines())
@@ -1340,7 +1349,13 @@
channels={"snapcraft": "stable"})
logger = BufferLogger()
- builds = getUtility(ISnapSet).makeAutoBuilds(logger=logger)
+ snapcraft_yaml = dedent("""\
+ architectures:
+ - build-on: %s
+ - build-on: %s
+ """) % (das1.architecturetag, das2.architecturetag)
+ with GitHostingFixture(blob=snapcraft_yaml):
+ builds = getUtility(ISnapSet).makeAutoBuilds(logger=logger)
self.assertThat(builds, MatchesSetwise(
MatchesStructure(
requester=Equals(snap1.owner), snap=Equals(snap1),
@@ -1369,14 +1384,19 @@
removeSecurityProxy(snap).is_stale = True
IStore(snap).flush()
logger = BufferLogger()
- builds = getUtility(ISnapSet).makeAutoBuilds(logger=logger)
+ with GitHostingFixture(blob=snapcraft_yaml):
+ builds = getUtility(ISnapSet).makeAutoBuilds(logger=logger)
self.assertEqual([], builds)
self.assertEqual([], logger.getLogBuffer().splitlines())
def test_makeAutoBuilds_skips_non_stale_snaps(self):
# ISnapSet.makeAutoBuilds skips snap packages that are not stale.
das, snap = self.makeAutoBuildableSnap(is_stale=False)
- self.assertEqual([], getUtility(ISnapSet).makeAutoBuilds())
+ with GitHostingFixture(blob=dedent("""\
+ architectures:
+ - build-on: %s
+ """) % das.architecturetag):
+ self.assertEqual([], getUtility(ISnapSet).makeAutoBuilds())
def test_makeAutoBuilds_skips_pending(self):
# ISnapSet.makeAutoBuilds skips snap packages that already have
@@ -1389,7 +1409,11 @@
distroarchseries=das,
date_created=datetime.now(pytz.UTC) - timedelta(days=1))
logger = BufferLogger()
- builds = getUtility(ISnapSet).makeAutoBuilds(logger=logger)
+ with GitHostingFixture(blob=dedent("""\
+ architectures:
+ - build-on: %s
+ """) % das.architecturetag):
+ builds = getUtility(ISnapSet).makeAutoBuilds(logger=logger)
self.assertEqual([], builds)
expected_log_entries = [
"DEBUG Scheduling builds of snap package %s/%s" % (
@@ -1410,7 +1434,11 @@
distroarchseries=das,
date_created=datetime.now(pytz.UTC) - timedelta(days=1),
status=BuildStatus.FULLYBUILT, duration=timedelta(minutes=1))
- builds = getUtility(ISnapSet).makeAutoBuilds()
+ with GitHostingFixture(blob=dedent("""\
+ architectures:
+ - build-on: %s
+ """) % das.architecturetag):
+ builds = getUtility(ISnapSet).makeAutoBuilds()
self.assertEqual(1, len(builds))
def test_makeAutoBuilds_with_older_and_newer_builds(self):
@@ -1423,7 +1451,11 @@
archive=snap.auto_build_archive, distroarchseries=das,
date_created=datetime.now(pytz.UTC) - timediff,
status=BuildStatus.FULLYBUILT, duration=timedelta(minutes=1))
- self.assertEqual([], getUtility(ISnapSet).makeAutoBuilds())
+ with GitHostingFixture(blob=dedent("""\
+ architectures:
+ - build-on: %s
+ """) % das.architecturetag):
+ self.assertEqual([], getUtility(ISnapSet).makeAutoBuilds())
def test_makeAutoBuilds_with_recent_build_from_different_archive(self):
# If a snap package has been built recently but from an archive
@@ -1434,9 +1466,49 @@
requester=snap.owner, snap=snap, distroarchseries=das,
date_created=datetime.now(pytz.UTC) - timedelta(minutes=30),
status=BuildStatus.FULLYBUILT, duration=timedelta(minutes=1))
- builds = getUtility(ISnapSet).makeAutoBuilds()
+ with GitHostingFixture(blob=dedent("""\
+ architectures:
+ - build-on: %s
+ """) % das.architecturetag):
+ builds = getUtility(ISnapSet).makeAutoBuilds()
self.assertEqual(1, len(builds))
+ def test_makeAutoBuilds_honours_explicit_architectures(self):
+ # ISnapSet.makeAutoBuilds honours an explicit list of architectures
+ # in the snap's snapcraft.yaml.
+ das1, snap = self.makeAutoBuildableSnap(is_stale=True)
+ das2 = self.makeBuildableDistroArchSeries(
+ distroseries=das1.distroseries,
+ processor=self.factory.makeProcessor(supports_virtualized=True))
+ das3 = self.makeBuildableDistroArchSeries(
+ distroseries=das1.distroseries,
+ processor=self.factory.makeProcessor(supports_virtualized=True))
+ snap.setProcessors([das1.processor, das2.processor, das3.processor])
+ logger = BufferLogger()
+ with GitHostingFixture(blob=dedent("""\
+ architectures:
+ - build-on: %s
+ - build-on: %s
+ """) % (das1.architecturetag, das2.architecturetag)):
+ builds = getUtility(ISnapSet).makeAutoBuilds(logger=logger)
+ self.assertThat(set(builds), MatchesSetwise(*[
+ MatchesStructure.byEquality(
+ requester=snap.owner, snap=snap, distro_arch_series=das,
+ status=BuildStatus.NEEDSBUILD)
+ for das in (das1, das2)
+ ]))
+ expected_log_entries = [
+ "DEBUG Scheduling builds of snap package %s/%s" % (
+ snap.owner.name, snap.name),
+ "DEBUG - %s/%s/%s: Build requested." % (
+ snap.owner.name, snap.name, das1.architecturetag),
+ "DEBUG - %s/%s/%s: Build requested." % (
+ snap.owner.name, snap.name, das2.architecturetag),
+ ]
+ self.assertEqual(
+ expected_log_entries, logger.getLogBuffer().splitlines())
+ self.assertFalse(snap.is_stale)
+
def test_detachFromBranch(self):
# ISnapSet.detachFromBranch clears the given Bazaar branch from all
# Snaps.
Follow ups
