launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #20644
[Merge] lp:~cjwatson/launchpad/snap-auto-build-model into lp:launchpad
Colin Watson has proposed merging lp:~cjwatson/launchpad/snap-auto-build-model into lp:launchpad.
Commit message:
Model new Snap columns to support auto-building.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
Related bugs:
Bug #1593359 in Launchpad itself: "Add option to trigger snap builds when top-level branch changes"
https://bugs.launchpad.net/launchpad/+bug/1593359
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/snap-auto-build-model/+merge/297955
Model new Snap columns to support auto-building.
See https://code.launchpad.net/~cjwatson/launchpad/db-snap-auto-build/+merge/297667 for the previous database branch.
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/snap-auto-build-model into lp:launchpad.
=== modified file 'lib/lp/services/config/schema-lazr.conf'
--- lib/lp/services/config/schema-lazr.conf 2016-05-25 15:31:48 +0000
+++ lib/lp/services/config/schema-lazr.conf 2016-06-20 20:33:26 +0000
@@ -1771,6 +1771,10 @@
http_proxy: none
[snappy]
+# Minimum time in minutes between dispatching automatic builds of snap
+# packages.
+auto_build_frequency: 60
+
# Admin secret for requesting tokens from the builder proxy service.
# For a mojo deployed proxy service, the secret is generated and will reside in
# /srv/mojo/${MOJO_PROJECT}/${SERIES}/${MOJO_WORKSPACE}/local/admin-api-secret
=== modified file 'lib/lp/snappy/interfaces/snap.py'
--- lib/lp/snappy/interfaces/snap.py 2016-05-28 00:21:40 +0000
+++ lib/lp/snappy/interfaces/snap.py 2016-06-20 20:33:26 +0000
@@ -377,6 +377,31 @@
"The Git branch containing a snapcraft.yaml recipe at the top "
"level.")))
+ auto_build = exported(Bool(
+ title=_("Automatically build when branch changes"),
+ required=True, readonly=False,
+ description=_(
+ "Whether this snap package is built automatically when the branch "
+ "containing its snapcraft.yaml recipe changes.")))
+
+ auto_build_archive = exported(Reference(
+ IArchive, title=_("Source archive for automatic builds"),
+ required=False, readonly=False,
+ description=_(
+ "The archive from which automatic builds of this snap package "
+ "should be built.")))
+
+ auto_build_pocket = exported(Choice(
+ title=_("Pocket for automatic builds"),
+ vocabulary=PackagePublishingPocket, required=False, readonly=False,
+ description=_(
+ "The pocket for which automatic builds of this snap package "
+ "should be built.")))
+
+ is_stale = Bool(
+ title=_("Snap package is stale and is due to be rebuilt."),
+ required=True, readonly=False)
+
store_upload = Bool(
title=_("Automatically upload to store"),
required=True, readonly=False,
@@ -523,6 +548,12 @@
def preloadDataForSnaps(snaps, user):
"""Load the data related to a list of snap packages."""
+ def makeAutoBuilds(logger=None):
+ """Create and return automatic builds for stale snap packages.
+
+ :param logger: An optional logger.
+ """
+
def detachFromBranch(branch):
"""Detach all snap packages from the given Bazaar branch.
=== modified file 'lib/lp/snappy/model/snap.py'
--- lib/lp/snappy/model/snap.py 2016-05-28 00:21:40 +0000
+++ lib/lp/snappy/model/snap.py 2016-06-20 20:33:26 +0000
@@ -6,14 +6,22 @@
'Snap',
]
+from datetime import (
+ datetime,
+ timedelta,
+ )
import pytz
+from storm.expr import (
+ And,
+ Desc,
+ LeftJoin,
+ Not,
+ )
from storm.locals import (
Bool,
DateTime,
- Desc,
Int,
JSON,
- Not,
Reference,
Store,
Storm,
@@ -56,6 +64,7 @@
IPerson,
IPersonSet,
)
+from lp.registry.interfaces.pocket import PackagePublishingPocket
from lp.registry.interfaces.product import IProduct
from lp.registry.interfaces.role import (
IHasOwner,
@@ -67,6 +76,7 @@
DEFAULT,
UTC_NOW,
)
+from lp.services.database.enumcol import DBEnum
from lp.services.database.interfaces import (
IMasterStore,
IStore,
@@ -155,6 +165,15 @@
git_path = Unicode(name='git_path', allow_none=True)
+ auto_build = Bool(name='auto_build', allow_none=False)
+
+ auto_build_archive_id = Int(name='auto_build_archive', allow_none=True)
+ auto_build_archive = Reference(auto_build_archive_id, 'Archive.id')
+
+ auto_build_pocket = DBEnum(enum=PackagePublishingPocket, allow_none=True)
+
+ is_stale = Bool(name='is_stale', allow_none=False)
+
require_virtualized = Bool(name='require_virtualized')
private = Bool(name='private')
@@ -169,7 +188,8 @@
store_secrets = JSON('store_secrets', allow_none=True)
def __init__(self, registrant, owner, distro_series, name,
- description=None, branch=None, git_ref=None,
+ description=None, branch=None, git_ref=None, auto_build=False,
+ auto_build_archive=None, auto_build_pocket=None,
require_virtualized=True, date_created=DEFAULT,
private=False, store_upload=False, store_series=None,
store_name=None, store_secrets=None):
@@ -185,6 +205,9 @@
self.description = description
self.branch = branch
self.git_ref = git_ref
+ self.auto_build = auto_build
+ self.auto_build_archive = auto_build_archive
+ self.auto_build_pocket = auto_build_pocket
self.require_virtualized = require_virtualized
self.date_created = date_created
self.date_last_modified = date_created
@@ -484,10 +507,11 @@
"""See `ISnapSet`."""
def new(self, registrant, owner, distro_series, name, description=None,
- branch=None, git_ref=None, require_virtualized=True,
- processors=None, date_created=DEFAULT, private=False,
- store_upload=False, store_series=None, store_name=None,
- store_secrets=None):
+ branch=None, git_ref=None, auto_build=False,
+ auto_build_archive=None, auto_build_pocket=None,
+ require_virtualized=True, processors=None, date_created=DEFAULT,
+ private=False, store_upload=False, store_series=None,
+ store_name=None, store_secrets=None):
"""See `ISnapSet`."""
if not registrant.inTeam(owner):
if owner.is_team:
@@ -510,7 +534,9 @@
store = IMasterStore(Snap)
snap = Snap(
registrant, owner, distro_series, name, description=description,
- branch=branch, git_ref=git_ref,
+ branch=branch, git_ref=git_ref, auto_build=auto_build,
+ auto_build_archive=auto_build_archive,
+ auto_build_pocket=auto_build_pocket,
require_virtualized=require_virtualized, date_created=date_created,
private=private, store_upload=store_upload,
store_series=store_series, store_name=store_name,
@@ -663,6 +689,55 @@
list(getUtility(IPersonSet).getPrecachedPersonsFromIDs(
person_ids, need_validity=True))
+ @staticmethod
+ def _findStaleSnaps():
+ """See `ISnapSet`."""
+ threshold_date = (
+ datetime.now(pytz.UTC) -
+ timedelta(minutes=config.snappy.auto_build_frequency))
+ origin = [
+ Snap,
+ LeftJoin(
+ SnapBuild,
+ And(
+ SnapBuild.snap_id == Snap.id,
+ SnapBuild.archive_id == Snap.auto_build_archive_id,
+ SnapBuild.pocket == Snap.auto_build_pocket,
+ # We only want Snaps that haven't had an automatic
+ # SnapBuild dispatched for them recently.
+ SnapBuild.date_created >= threshold_date)),
+ ]
+ return IStore(Snap).using(*origin).find(
+ Snap,
+ Snap.is_stale == True,
+ Snap.auto_build == True,
+ SnapBuild.date_created == None).config(distinct=True)
+
+ @classmethod
+ def makeAutoBuilds(cls, logger=None):
+ """See `ISnapSet`."""
+ snaps = cls._findStaleSnaps()
+ builds = []
+ for snap in snaps:
+ snap.is_stale = False
+ if logger is not None:
+ logger.debug(
+ "Scheduling builds of snap package %s/%s",
+ snap.owner.name, snap.name)
+ for arch in snap.getAllowedArchitectures():
+ try:
+ build = snap.requestBuild(
+ snap.owner, snap.auto_build_archive, arch,
+ snap.auto_build_pocket)
+ if logger is not None:
+ logger.debug(
+ " - %s: Build requested.", arch.architecturetag)
+ builds.append(build)
+ except Exception as e:
+ if logger is not None:
+ logger.debug(" - %s: %s", arch.architecturetag, e)
+ return builds
+
def detachFromBranch(self, branch):
"""See `ISnapSet`."""
self.findByBranch(branch).set(
=== modified file 'lib/lp/snappy/tests/test_snap.py'
--- lib/lp/snappy/tests/test_snap.py 2016-05-17 12:47:34 +0000
+++ lib/lp/snappy/tests/test_snap.py 2016-06-20 20:33:26 +0000
@@ -5,12 +5,19 @@
__metaclass__ = type
-from datetime import timedelta
+from datetime import (
+ datetime,
+ timedelta,
+ )
from lazr.lifecycle.event import ObjectModifiedEvent
+import pytz
from storm.exceptions import LostObjectError
from storm.locals import Store
-from testtools.matchers import Equals
+from testtools.matchers import (
+ Equals,
+ MatchesStructure,
+ )
import transaction
from zope.component import getUtility
from zope.event import notify
@@ -35,6 +42,7 @@
)
from lp.services.database.sqlbase import flush_database_caches
from lp.services.features.testing import FeatureFixture
+from lp.services.log.logger import BufferLogger
from lp.services.propertycache import clear_property_cache
from lp.services.webapp.interfaces import OAuthPermission
from lp.snappy.interfaces.snap import (
@@ -56,6 +64,7 @@
ISnapBuild,
ISnapBuildSet,
)
+from lp.snappy.model.snap import SnapSet
from lp.snappy.model.snapbuild import SnapFile
from lp.testing import (
admin_logged_in,
@@ -372,7 +381,7 @@
build11 = self.factory.makeSnapBuild(snap=snap1)
build12 = self.factory.makeSnapBuild(snap=snap1)
build2 = self.factory.makeSnapBuild(snap=snap2)
- build3 = self.factory.makeSnapBuild()
+ self.factory.makeSnapBuild()
summary1 = snap1.getBuildSummariesForSnapBuildIds(
[build11.id, build12.id])
summary2 = snap2.getBuildSummariesForSnapBuildIds([build2.id])
@@ -391,7 +400,7 @@
# Should not return build summaries of other snaps.
snap1 = self.factory.makeSnap()
snap2 = self.factory.makeSnap()
- build1 = self.factory.makeSnapBuild(snap=snap1)
+ self.factory.makeSnapBuild(snap=snap1)
build2 = self.factory.makeSnapBuild(snap=snap2)
summary1 = snap1.getBuildSummariesForSnapBuildIds([build2.id])
self.assertEqual({}, summary1)
@@ -539,6 +548,9 @@
self.assertIsNone(snap.git_repository)
self.assertIsNone(snap.git_path)
self.assertIsNone(snap.git_ref)
+ self.assertFalse(snap.auto_build)
+ self.assertIsNone(snap.auto_build_archive)
+ self.assertIsNone(snap.auto_build_pocket)
self.assertTrue(snap.require_virtualized)
self.assertFalse(snap.private)
@@ -556,6 +568,9 @@
self.assertEqual(ref.repository, snap.git_repository)
self.assertEqual(ref.path, snap.git_path)
self.assertEqual(ref, snap.git_ref)
+ self.assertFalse(snap.auto_build)
+ self.assertIsNone(snap.auto_build_archive)
+ self.assertIsNone(snap.auto_build_pocket)
self.assertTrue(snap.require_virtualized)
self.assertFalse(snap.private)
@@ -750,6 +765,144 @@
BadSnapSearchContext, snap_set.findByContext,
self.factory.makeDistribution())
+ def test__findStaleSnaps(self):
+ # Stale; not built automatically.
+ self.factory.makeSnap(is_stale=True)
+ # Not stale; built automatically.
+ self.factory.makeSnap(auto_build=True, is_stale=False)
+ # Stale; built automatically.
+ stale_daily = self.factory.makeSnap(auto_build=True, is_stale=True)
+ self.assertContentEqual([stale_daily], SnapSet._findStaleSnaps())
+
+ def test__findStaleSnapsDistinct(self):
+ # If a snap package has two builds due to two architectures, it only
+ # returns one recipe.
+ distroseries = self.factory.makeDistroSeries()
+ dases = [
+ self.factory.makeDistroArchSeries(distroseries=distroseries)
+ for _ in range(2)]
+ snap = self.factory.makeSnap(
+ distroseries=distroseries,
+ processors=[das.processor for das in dases],
+ auto_build=True, is_stale=True)
+ for das in dases:
+ self.factory.makeSnapBuild(
+ requester=snap.owner, snap=snap,
+ archive=snap.auto_build_archive, distroarchseries=das,
+ pocket=snap.auto_build_pocket,
+ date_created=(datetime.now(pytz.UTC) - timedelta(days=2)))
+ self.assertContentEqual([snap], SnapSet._findStaleSnaps())
+
+ def makeBuildableDistroArchSeries(self, **kwargs):
+ das = self.factory.makeDistroArchSeries(**kwargs)
+ fake_chroot = self.factory.makeLibraryFileAlias(
+ filename="fake_chroot.tar.gz", db_only=True)
+ das.addOrUpdateChroot(fake_chroot)
+ return das
+
+ def makeAutoBuildableSnap(self, **kwargs):
+ processor = self.factory.makeProcessor(supports_virtualized=True)
+ das = self.makeBuildableDistroArchSeries(processor=processor)
+ snap = self.factory.makeSnap(
+ distroseries=das.distroseries, processors=[das.processor],
+ auto_build=True, **kwargs)
+ return das, snap
+
+ def test_makeAutoBuilds(self):
+ # ISnapSet.makeAutoBuilds requests builds of
+ # appropriately-configured Snaps where possible.
+ self.assertEqual([], getUtility(ISnapSet).makeAutoBuilds())
+ das, snap = self.makeAutoBuildableSnap(is_stale=True)
+ logger = BufferLogger()
+ [build] = getUtility(ISnapSet).makeAutoBuilds(logger=logger)
+ self.assertThat(build, MatchesStructure.byEquality(
+ requester=snap.owner, snap=snap, distro_arch_series=das,
+ status=BuildStatus.NEEDSBUILD,
+ ))
+ expected_log_entries = [
+ "DEBUG Scheduling builds of snap package %s/%s" % (
+ snap.owner.name, snap.name),
+ "DEBUG - %s: Build requested." % das.architecturetag,
+ ]
+ self.assertEqual(
+ expected_log_entries, logger.getLogBuffer().splitlines())
+ self.assertFalse(snap.is_stale)
+
+ def test_makeAutoBuilds_skips_if_built_recently(self):
+ # ISnapSet.makeAutoBuilds skips snap packages that have been built
+ # recently.
+ das, snap = self.makeAutoBuildableSnap(is_stale=True)
+ self.factory.makeSnapBuild(
+ requester=snap.owner, snap=snap, archive=snap.auto_build_archive,
+ distroarchseries=das)
+ logger = BufferLogger()
+ builds = getUtility(ISnapSet).makeAutoBuilds(logger=logger)
+ self.assertEqual([], builds)
+ self.assertEqual([], logger.getLogBuffer().splitlines())
+
+ def test_makeAutoBuilds_skips_non_stale_snaps(self):
+ # ISnapSet.makeAutoBuilds skips snap packages that are not stale.
+ das, snap = self.makeAutoBuildableSnap(is_stale=False)
+ self.assertEqual([], getUtility(ISnapSet).makeAutoBuilds())
+
+ def test_makeAutoBuilds_skips_pending(self):
+ # ISnapSet.makeAutoBuilds skips snap packages that already have
+ # pending builds.
+ das, snap = self.makeAutoBuildableSnap(is_stale=True)
+ # Simulate very long build farm queues so that this case isn't
+ # filtered out earlier.
+ self.factory.makeSnapBuild(
+ requester=snap.owner, snap=snap, archive=snap.auto_build_archive,
+ distroarchseries=das,
+ date_created=datetime.now(pytz.UTC) - timedelta(days=1))
+ logger = BufferLogger()
+ builds = getUtility(ISnapSet).makeAutoBuilds(logger=logger)
+ self.assertEqual([], builds)
+ expected_log_entries = [
+ "DEBUG Scheduling builds of snap package %s/%s" % (
+ snap.owner.name, snap.name),
+ "DEBUG - %s: An identical build of this snap package is already "
+ "pending." % das.architecturetag,
+ ]
+ self.assertEqual(
+ expected_log_entries, logger.getLogBuffer().splitlines())
+
+ def test_makeAutoBuilds_with_older_build(self):
+ # If a previous build is not recent and the snap package is stale,
+ # ISnapSet.makeAutoBuilds requests builds.
+ das, snap = self.makeAutoBuildableSnap(is_stale=True)
+ self.factory.makeSnapBuild(
+ requester=snap.owner, snap=snap, archive=snap.auto_build_archive,
+ distroarchseries=das,
+ date_created=datetime.now(pytz.UTC) - timedelta(days=1),
+ status=BuildStatus.FULLYBUILT, duration=timedelta(minutes=1))
+ builds = getUtility(ISnapSet).makeAutoBuilds()
+ self.assertEqual(1, len(builds))
+
+ def test_makeAutoBuilds_with_older_and_newer_builds(self):
+ # If a snap package has been built twice, and the most recent build
+ # is too recent, ISnapSet.makeAutoBuilds does not request builds.
+ das, snap = self.makeAutoBuildableSnap(is_stale=True)
+ for timediff in timedelta(days=1), timedelta(minutes=30):
+ self.factory.makeSnapBuild(
+ requester=snap.owner, snap=snap,
+ archive=snap.auto_build_archive, distroarchseries=das,
+ date_created=datetime.now(pytz.UTC) - timediff,
+ status=BuildStatus.FULLYBUILT, duration=timedelta(minutes=1))
+ self.assertEqual([], getUtility(ISnapSet).makeAutoBuilds())
+
+ def test_makeAutoBuilds_with_recent_build_from_different_archive(self):
+ # If a snap package has been built recently but from an archive
+ # other than the auto_build_archive, ISnapSet.makeAutoBuilds
+ # requests builds.
+ das, snap = self.makeAutoBuildableSnap(is_stale=True)
+ self.factory.makeSnapBuild(
+ requester=snap.owner, snap=snap, distroarchseries=das,
+ date_created=datetime.now(pytz.UTC) - timedelta(minutes=30),
+ status=BuildStatus.FULLYBUILT, duration=timedelta(minutes=1))
+ builds = getUtility(ISnapSet).makeAutoBuilds()
+ self.assertEqual(1, len(builds))
+
def test_detachFromBranch(self):
# ISnapSet.detachFromBranch clears the given Bazaar branch from all
# Snaps.
=== modified file 'lib/lp/testing/factory.py'
--- lib/lp/testing/factory.py 2016-06-17 15:46:57 +0000
+++ lib/lp/testing/factory.py 2016-06-20 20:33:26 +0000
@@ -4630,8 +4630,9 @@
active, secret)
def makeSnap(self, registrant=None, owner=None, distroseries=None,
- name=None, branch=None, git_ref=None,
- require_virtualized=True, processors=None,
+ name=None, branch=None, git_ref=None, auto_build=False,
+ auto_build_archive=None, auto_build_pocket=None,
+ is_stale=None, require_virtualized=True, processors=None,
date_created=DEFAULT, private=False, store_upload=False,
store_series=None, store_name=None, store_secrets=None):
"""Make a new Snap."""
@@ -4645,13 +4646,22 @@
name = self.getUniqueString(u"snap-name")
if branch is None and git_ref is None:
branch = self.makeAnyBranch()
+ if auto_build:
+ if auto_build_archive is None:
+ auto_build_archive = self.makeArchive(
+ distribution=distroseries.distribution, owner=owner)
+ if auto_build_pocket is None:
+ auto_build_pocket = PackagePublishingPocket.UPDATES
snap = getUtility(ISnapSet).new(
registrant, owner, distroseries, name,
require_virtualized=require_virtualized, processors=processors,
date_created=date_created, branch=branch, git_ref=git_ref,
- private=private, store_upload=store_upload,
- store_series=store_series, store_name=store_name,
- store_secrets=store_secrets)
+ auto_build=auto_build, auto_build_archive=auto_build_archive,
+ auto_build_pocket=auto_build_pocket, private=private,
+ store_upload=store_upload, store_series=store_series,
+ store_name=store_name, store_secrets=store_secrets)
+ if is_stale is not None:
+ removeSecurityProxy(snap).is_stale = is_stale
IStore(snap).flush()
return snap
Follow ups
