launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #29985
[Merge] ~cjwatson/launchpad:ci-build-webhooks into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:ci-build-webhooks into launchpad:master.
Commit message:
Add webhooks for CI builds
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/442596
This was mostly just cribbed from charm recipe build webhooks, with adjustments as needed for the different data model here. I expect we'll probably need to add some more fields to the payload once people start using this in practice, but this should be enough to get started.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:ci-build-webhooks into launchpad:master.
diff --git a/lib/lp/code/configure.zcml b/lib/lp/code/configure.zcml
index eaf4020..cf2e52e 100644
--- a/lib/lp/code/configure.zcml
+++ b/lib/lp/code/configure.zcml
@@ -1,4 +1,4 @@
-<!-- Copyright 2009-2021 Canonical Ltd. This software is licensed under the
+<!-- Copyright 2009-2023 Canonical Ltd. This software is licensed under the
GNU Affero General Public License version 3 (see the file LICENSE).
-->
@@ -1297,6 +1297,14 @@
permission="launchpad.Admin"
interface="lp.code.interfaces.cibuild.ICIBuildAdmin" />
</class>
+ <subscriber
+ for="lp.code.interfaces.cibuild.ICIBuild
+ lazr.lifecycle.interfaces.IObjectCreatedEvent"
+ handler="lp.code.subscribers.cibuild.ci_build_created" />
+ <subscriber
+ for="lp.code.interfaces.cibuild.ICIBuild
+ lazr.lifecycle.interfaces.IObjectModifiedEvent"
+ handler="lp.code.subscribers.cibuild.ci_build_modified" />
<!-- CIBuildSet -->
<lp:securedutility
diff --git a/lib/lp/code/interfaces/cibuild.py b/lib/lp/code/interfaces/cibuild.py
index 9599b75..bac6181 100644
--- a/lib/lp/code/interfaces/cibuild.py
+++ b/lib/lp/code/interfaces/cibuild.py
@@ -1,9 +1,10 @@
-# Copyright 2022 Canonical Ltd. This software is licensed under the
+# Copyright 2022-2023 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Interfaces for CI builds."""
__all__ = [
+ "CI_WEBHOOKS_FEATURE_FLAG",
"CannotFetchConfiguration",
"CannotParseConfiguration",
"CIBuildAlreadyRequested",
@@ -41,6 +42,8 @@ from lp.code.interfaces.gitrepository import IGitRepository
from lp.services.database.constants import DEFAULT
from lp.soyuz.interfaces.distroarchseries import IDistroArchSeries
+CI_WEBHOOKS_FEATURE_FLAG = "ci.webhooks.enabled"
+
class MissingConfiguration(Exception):
"""The repository for this CI build does not have a .launchpad.yaml."""
diff --git a/lib/lp/code/model/cibuild.py b/lib/lp/code/model/cibuild.py
index ad5bbfb..e1b7450 100644
--- a/lib/lp/code/model/cibuild.py
+++ b/lib/lp/code/model/cibuild.py
@@ -1,4 +1,4 @@
-# Copyright 2022 Canonical Ltd. This software is licensed under the
+# Copyright 2022-2023 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""CI builds."""
@@ -77,6 +77,7 @@ from lp.services.macaroons.interfaces import (
)
from lp.services.macaroons.model import MacaroonIssuerBase
from lp.services.propertycache import cachedproperty
+from lp.services.webapp.snapshot import notify_modified
from lp.soyuz.model.binarypackagename import BinaryPackageName
from lp.soyuz.model.binarypackagerelease import BinaryPackageRelease
from lp.soyuz.model.distroarchseries import DistroArchSeries
@@ -504,6 +505,31 @@ class CIBuild(PackageBuildMixin, StormBase):
# We have no interesting checks to perform here.
return True
+ def updateStatus(
+ self,
+ status,
+ builder=None,
+ worker_status=None,
+ date_started=None,
+ date_finished=None,
+ force_invalid_transition=False,
+ ):
+ """See `IBuildFarmJob`."""
+ edited_fields = set()
+ with notify_modified(
+ self, edited_fields, snapshot_names=("status",)
+ ) as previous_obj:
+ super().updateStatus(
+ status,
+ builder=builder,
+ worker_status=worker_status,
+ date_started=date_started,
+ date_finished=date_finished,
+ force_invalid_transition=force_invalid_transition,
+ )
+ if self.status != previous_obj.status:
+ edited_fields.add("status")
+
def notify(self, extra_info=None):
"""See `IPackageBuild`."""
from lp.code.mail.cibuild import CIBuildMailer
diff --git a/lib/lp/code/model/gitrepository.py b/lib/lp/code/model/gitrepository.py
index 0ffae31..0384d58 100644
--- a/lib/lp/code/model/gitrepository.py
+++ b/lib/lp/code/model/gitrepository.py
@@ -417,7 +417,7 @@ class GitRepository(
@property
def valid_webhook_event_types(self):
- return ["git:push:0.1", "merge-proposal:0.1"]
+ return ["ci:build:0.1", "git:push:0.1", "merge-proposal:0.1"]
@property
def default_webhook_event_types(self):
diff --git a/lib/lp/code/model/tests/test_cibuild.py b/lib/lp/code/model/tests/test_cibuild.py
index ec452d6..ae95fbd 100644
--- a/lib/lp/code/model/tests/test_cibuild.py
+++ b/lib/lp/code/model/tests/test_cibuild.py
@@ -1,4 +1,4 @@
-# Copyright 2022 Canonical Ltd. This software is licensed under the
+# Copyright 2022-2023 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Test CI builds."""
@@ -9,13 +9,14 @@ from textwrap import dedent
from unittest.mock import Mock
from urllib.request import urlopen
-from fixtures import MockPatchObject
+from fixtures import FakeLogger, MockPatchObject
from pymacaroons import Macaroon
from storm.locals import Store
from testtools.matchers import (
ContainsDict,
Equals,
Is,
+ MatchesDict,
MatchesListwise,
MatchesSetwise,
MatchesStructure,
@@ -35,6 +36,7 @@ from lp.buildmaster.model.buildfarmjob import BuildFarmJob
from lp.buildmaster.model.buildqueue import BuildQueue
from lp.code.errors import GitRepositoryBlobNotFound, GitRepositoryScanFault
from lp.code.interfaces.cibuild import (
+ CI_WEBHOOKS_FEATURE_FLAG,
CannotFetchConfiguration,
CannotParseConfiguration,
CIBuildAlreadyRequested,
@@ -55,12 +57,15 @@ from lp.registry.interfaces.sourcepackage import SourcePackageType
from lp.services.authserver.xmlrpc import AuthServerAPIView
from lp.services.config import config
from lp.services.database.sqlbase import flush_database_caches
+from lp.services.features.testing import FeatureFixture
from lp.services.librarian.browser import ProxiedLibraryFileAlias
from lp.services.log.logger import BufferLogger
from lp.services.macaroons.interfaces import IMacaroonIssuer
from lp.services.macaroons.testing import MacaroonTestMixin
from lp.services.propertycache import clear_property_cache
from lp.services.webapp.interfaces import OAuthPermission
+from lp.services.webapp.publisher import canonical_url
+from lp.services.webhooks.testing import LogsScheduledWebhooks
from lp.soyuz.enums import BinaryPackageFormat
from lp.testing import (
ANONYMOUS,
@@ -73,6 +78,7 @@ from lp.testing import (
person_logged_in,
pop_notifications,
)
+from lp.testing.dbuser import dbuser
from lp.testing.layers import LaunchpadFunctionalLayer, LaunchpadZopelessLayer
from lp.testing.matchers import HasQueryCount
from lp.testing.pages import webservice_for_person
@@ -320,6 +326,95 @@ class TestCIBuild(TestCaseWithFactory):
build = self.factory.makeCIBuild()
self.assertTrue(build.verifySuccessfulUpload())
+ def test_updateStatus_triggers_webhooks(self):
+ # Updating the status of a CIBuild triggers webhooks on the
+ # corresponding GitRepository.
+ self.useFixture(FeatureFixture({CI_WEBHOOKS_FEATURE_FLAG: "on"}))
+ logger = self.useFixture(FakeLogger())
+ build = self.factory.makeCIBuild()
+ hook = self.factory.makeWebhook(
+ target=build.git_repository, event_types=["ci:build:0.1"]
+ )
+ build.updateStatus(BuildStatus.FULLYBUILT)
+ expected_payload = {
+ "build": Equals(canonical_url(build, force_local_path=True)),
+ "action": Equals("status-changed"),
+ "git_repository": Equals(
+ canonical_url(build.git_repository, force_local_path=True)
+ ),
+ "commit_sha1": Equals(build.commit_sha1),
+ "status": Equals("Successfully built"),
+ }
+ delivery = hook.deliveries.one()
+ self.assertThat(
+ delivery,
+ MatchesStructure(
+ event_type=Equals("ci:build:0.1"),
+ payload=MatchesDict(expected_payload),
+ ),
+ )
+ with dbuser(config.IWebhookDeliveryJobSource.dbuser):
+ self.assertEqual(
+ "<WebhookDeliveryJob for webhook %d on %r>"
+ % (hook.id, hook.target),
+ repr(delivery),
+ )
+ self.assertThat(
+ logger.output,
+ LogsScheduledWebhooks(
+ [(hook, "ci:build:0.1", MatchesDict(expected_payload))]
+ ),
+ )
+
+ def test_updateStatus_no_change_does_not_trigger_webhooks(self):
+ # An updateStatus call that changes details of the worker status but
+ # that doesn't change the build's status attribute does not trigger
+ # webhooks.
+ self.useFixture(FeatureFixture({CI_WEBHOOKS_FEATURE_FLAG: "on"}))
+ logger = self.useFixture(FakeLogger())
+ build = self.factory.makeCIBuild()
+ hook = self.factory.makeWebhook(
+ target=build.git_repository, event_types=["ci:build:0.1"]
+ )
+ builder = self.factory.makeBuilder()
+ build.updateStatus(BuildStatus.BUILDING)
+ expected_logs = [
+ (
+ hook,
+ "ci:build:0.1",
+ ContainsDict(
+ {
+ "action": Equals("status-changed"),
+ "status": Equals("Currently building"),
+ }
+ ),
+ )
+ ]
+ self.assertEqual(1, hook.deliveries.count())
+ self.assertThat(logger.output, LogsScheduledWebhooks(expected_logs))
+ build.updateStatus(
+ BuildStatus.BUILDING,
+ builder=builder,
+ worker_status={"revision_id": build.commit_sha1},
+ )
+ self.assertEqual(1, hook.deliveries.count())
+ self.assertThat(logger.output, LogsScheduledWebhooks(expected_logs))
+ build.updateStatus(BuildStatus.UPLOADING)
+ expected_logs.append(
+ (
+ hook,
+ "ci:build:0.1",
+ ContainsDict(
+ {
+ "action": Equals("status-changed"),
+ "status": Equals("Uploading build"),
+ }
+ ),
+ )
+ )
+ self.assertEqual(2, hook.deliveries.count())
+ self.assertThat(logger.output, LogsScheduledWebhooks(expected_logs))
+
def addFakeBuildLog(self, build):
build.setLog(self.factory.makeLibraryFileAlias("mybuildlog.txt"))
diff --git a/lib/lp/code/model/tests/test_gitrepository.py b/lib/lp/code/model/tests/test_gitrepository.py
index 2aeb2ca..ecf5663 100644
--- a/lib/lp/code/model/tests/test_gitrepository.py
+++ b/lib/lp/code/model/tests/test_gitrepository.py
@@ -12,7 +12,7 @@ from textwrap import dedent
import transaction
from breezy import urlutils
-from fixtures import MockPatch
+from fixtures import FakeLogger, MockPatch
from lazr.lifecycle.event import ObjectModifiedEvent
from pymacaroons import Macaroon
from storm.exceptions import LostObjectError
@@ -76,7 +76,11 @@ from lp.code.event.git import GitRefsUpdatedEvent
from lp.code.interfaces.branchmergeproposal import (
BRANCH_MERGE_PROPOSAL_FINAL_STATES as FINAL_STATES,
)
-from lp.code.interfaces.cibuild import ICIBuild, ICIBuildSet
+from lp.code.interfaces.cibuild import (
+ CI_WEBHOOKS_FEATURE_FLAG,
+ ICIBuild,
+ ICIBuildSet,
+)
from lp.code.interfaces.codeimport import ICodeImportSet
from lp.code.interfaces.defaultgit import ICanHasDefaultGitRepository
from lp.code.interfaces.gitjob import (
@@ -170,7 +174,9 @@ from lp.services.propertycache import clear_property_cache
from lp.services.utils import seconds_since_epoch
from lp.services.webapp.authorization import check_permission
from lp.services.webapp.interfaces import OAuthPermission
+from lp.services.webapp.publisher import canonical_url
from lp.services.webapp.snapshot import notify_modified
+from lp.services.webhooks.testing import LogsScheduledWebhooks
from lp.snappy.interfaces.snap import SNAP_TESTING_FLAGS
from lp.testing import (
ANONYMOUS,
@@ -4171,6 +4177,83 @@ class TestGitRepositoryRequestCIBuilds(TestCaseWithFactory):
)
self.assertEqual("", logger.getLogBuffer())
+ def test_triggers_webhooks(self):
+ # Requesting CI builds triggers any relevant webhooks.
+ self.useFixture(FeatureFixture({CI_WEBHOOKS_FEATURE_FLAG: "on"}))
+ logger = self.useFixture(FakeLogger())
+ repository = self.factory.makeGitRepository()
+ hook = self.factory.makeWebhook(
+ target=repository, event_types=["ci:build:0.1"]
+ )
+ ubuntu = getUtility(ILaunchpadCelebrities).ubuntu
+ distroseries = self.factory.makeDistroSeries(distribution=ubuntu)
+ das = self.factory.makeBuildableDistroArchSeries(
+ distroseries=distroseries
+ )
+ configuration = dedent(
+ """\
+ pipeline: [test]
+ jobs:
+ test:
+ series: {series}
+ architectures: [{architecture}]
+ """.format(
+ series=distroseries.name, architecture=das.architecturetag
+ )
+ ).encode()
+ new_commit = hashlib.sha1(self.factory.getUniqueBytes()).hexdigest()
+ self.useFixture(
+ GitHostingFixture(
+ commits=[
+ {
+ "sha1": new_commit,
+ "blobs": {".launchpad.yaml": configuration},
+ },
+ ]
+ )
+ )
+ with dbuser("branchscanner"):
+ repository.createOrUpdateRefs(
+ {
+ "refs/heads/test": {
+ "sha1": new_commit,
+ "type": GitObjectType.COMMIT,
+ }
+ }
+ )
+
+ [build] = getUtility(ICIBuildSet).findByGitRepository(repository)
+ delivery = hook.deliveries.one()
+ payload_matcher = MatchesDict(
+ {
+ "build": Equals(canonical_url(build, force_local_path=True)),
+ "action": Equals("created"),
+ "git_repository": Equals(
+ canonical_url(repository, force_local_path=True)
+ ),
+ "commit_sha1": Equals(new_commit),
+ "status": Equals("Needs building"),
+ }
+ )
+ self.assertThat(
+ delivery,
+ MatchesStructure(
+ event_type=Equals("ci:build:0.1"), payload=payload_matcher
+ ),
+ )
+ with dbuser(config.IWebhookDeliveryJobSource.dbuser):
+ self.assertEqual(
+ "<WebhookDeliveryJob for webhook %d on %r>"
+ % (hook.id, hook.target),
+ repr(delivery),
+ )
+ self.assertThat(
+ logger.output,
+ LogsScheduledWebhooks(
+ [(hook, "ci:build:0.1", payload_matcher)]
+ ),
+ )
+
class TestGitRepositoryGetBlob(TestCaseWithFactory):
"""Tests for retrieving files from a Git repository."""
diff --git a/lib/lp/code/subscribers/cibuild.py b/lib/lp/code/subscribers/cibuild.py
new file mode 100644
index 0000000..a0eb3fc
--- /dev/null
+++ b/lib/lp/code/subscribers/cibuild.py
@@ -0,0 +1,40 @@
+# Copyright 2023 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Event subscribers for CI builds."""
+
+from zope.component import getUtility
+
+from lp.code.interfaces.cibuild import CI_WEBHOOKS_FEATURE_FLAG, ICIBuild
+from lp.services.features import getFeatureFlag
+from lp.services.webapp.publisher import canonical_url
+from lp.services.webhooks.interfaces import IWebhookSet
+from lp.services.webhooks.payload import compose_webhook_payload
+
+
+def _trigger_ci_build_webhook(build, action):
+ if getFeatureFlag(CI_WEBHOOKS_FEATURE_FLAG):
+ payload = {
+ "build": canonical_url(build, force_local_path=True),
+ "action": action,
+ }
+ payload.update(
+ compose_webhook_payload(
+ ICIBuild, build, ["git_repository", "commit_sha1", "status"]
+ )
+ )
+ getUtility(IWebhookSet).trigger(
+ build.git_repository, "ci:build:0.1", payload
+ )
+
+
+def ci_build_created(build, event):
+ """Trigger events when a new CI build is created."""
+ _trigger_ci_build_webhook(build, "created")
+
+
+def ci_build_modified(build, event):
+ """Trigger events when a CI build is modified."""
+ if event.edited_fields is not None:
+ if "status" in event.edited_fields:
+ _trigger_ci_build_webhook(build, "status-changed")
diff --git a/lib/lp/services/webhooks/interfaces.py b/lib/lp/services/webhooks/interfaces.py
index b31f29d..4f0a1e8 100644
--- a/lib/lp/services/webhooks/interfaces.py
+++ b/lib/lp/services/webhooks/interfaces.py
@@ -1,4 +1,4 @@
-# Copyright 2015-2021 Canonical Ltd. This software is licensed under the
+# Copyright 2015-2023 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Webhook interfaces."""
@@ -50,6 +50,7 @@ from lp.services.webservice.apihelpers import (
WEBHOOK_EVENT_TYPES = {
"bzr:push:0.1": "Bazaar push",
"charm-recipe:build:0.1": "Charm recipe build",
+ "ci:build:0.1": "CI build",
"git:push:0.1": "Git push",
"livefs:build:0.1": "Live filesystem build",
"merge-proposal:0.1": "Merge proposal",
diff --git a/lib/lp/services/webhooks/tests/test_browser.py b/lib/lp/services/webhooks/tests/test_browser.py
index 65ba0c3..b3f28fe 100644
--- a/lib/lp/services/webhooks/tests/test_browser.py
+++ b/lib/lp/services/webhooks/tests/test_browser.py
@@ -1,4 +1,4 @@
-# Copyright 2015-2021 Canonical Ltd. This software is licensed under the
+# Copyright 2015-2023 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Unit tests for Webhook views."""
@@ -63,6 +63,7 @@ class GitRepositoryTestHelpers:
event_type = "git:push:0.1"
expected_event_types = [
+ ("ci:build:0.1", "CI build"),
("git:push:0.1", "Git push"),
("merge-proposal:0.1", "Merge proposal"),
]
