launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #28440
[Merge] ~cjwatson/launchpad:ci-build-api into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:ci-build-api into launchpad:master.
Commit message:
Export CIBuild on the webservice API
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/422031
This should be enough to traverse from revision status reports to any associated CI builds, to perform build farm management tasks such as cancelling/retrying/rescoring builds, and to fetch any files associated with CI builds.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:ci-build-api into launchpad:master.
diff --git a/lib/lp/code/interfaces/cibuild.py b/lib/lp/code/interfaces/cibuild.py
index ed9dedf..7cd0d87 100644
--- a/lib/lp/code/interfaces/cibuild.py
+++ b/lib/lp/code/interfaces/cibuild.py
@@ -15,7 +15,13 @@ __all__ = [
import http.client
-from lazr.restful.declarations import error_status
+from lazr.restful.declarations import (
+ error_status,
+ export_read_operation,
+ exported,
+ exported_as_webservice_entry,
+ operation_for_version,
+ )
from lazr.restful.fields import Reference
from zope.schema import (
Bool,
@@ -84,27 +90,27 @@ class CIBuildAlreadyRequested(Exception):
class ICIBuildView(IPackageBuildView, IPrivacy):
"""`ICIBuild` attributes that require launchpad.View."""
- git_repository = Reference(
+ git_repository = exported(Reference(
IGitRepository,
title=_("The Git repository for this CI build."),
- required=False, readonly=True)
+ required=False, readonly=True))
- commit_sha1 = TextLine(
+ commit_sha1 = exported(TextLine(
title=_("The Git commit ID for this CI build."),
- required=True, readonly=True)
+ required=True, readonly=True))
- distro_arch_series = Reference(
+ distro_arch_series = exported(Reference(
IDistroArchSeries,
title=_(
"The series and architecture that this CI build should run on."),
- required=True, readonly=True)
+ required=True, readonly=True))
- arch_tag = TextLine(
- title=_("Architecture tag"), required=True, readonly=True)
+ arch_tag = exported(TextLine(
+ title=_("Architecture tag"), required=True, readonly=True))
- score = Int(
+ score = exported(Int(
title=_("Score of the related build farm job (if any)."),
- required=False, readonly=True)
+ required=False, readonly=True))
eta = Datetime(
title=_("The datetime when the build job is estimated to complete."),
@@ -118,13 +124,13 @@ class ICIBuildView(IPackageBuildView, IPrivacy):
"The date when the build completed or is estimated to complete."),
readonly=True)
- stages = List(
- title=_("A list of stages in this build's configured pipeline."))
+ stages = exported(List(
+ title=_("A list of stages in this build's configured pipeline.")))
- results = Dict(
+ results = exported(Dict(
title=_(
"A mapping from job IDs to result tokens, retrieved from the "
- "builder."))
+ "builder.")))
def getConfiguration(logger=None):
"""Fetch a CI build's .launchpad.yaml from code hosting, if possible.
@@ -163,6 +169,14 @@ class ICIBuildView(IPackageBuildView, IPrivacy):
:return: The corresponding `ILibraryFileAlias`.
"""
+ @export_read_operation()
+ @operation_for_version("devel")
+ def getFileUrls():
+ """URLs for all the files produced by this build.
+
+ :return: A collection of URLs for this build.
+ """
+
class ICIBuildEdit(IBuildFarmJobEdit):
"""`ICIBuild` methods that require launchpad.Edit."""
@@ -172,6 +186,7 @@ class ICIBuildAdmin(IBuildFarmJobAdmin):
"""`ICIBuild` methods that require launchpad.Admin."""
+@exported_as_webservice_entry(as_of="devel", singular_name="ci_build")
class ICIBuild(ICIBuildView, ICIBuildEdit, ICIBuildAdmin, IPackageBuild):
"""A build record for a pipeline of CI jobs."""
diff --git a/lib/lp/code/interfaces/revisionstatus.py b/lib/lp/code/interfaces/revisionstatus.py
index 708cc40..cf9c856 100644
--- a/lib/lp/code/interfaces/revisionstatus.py
+++ b/lib/lp/code/interfaces/revisionstatus.py
@@ -124,10 +124,10 @@ class IRevisionStatusReportEditableAttributes(Interface):
title=_('Result of the report'), readonly=True,
required=False, vocabulary=RevisionStatusResult))
- ci_build = Reference(
+ ci_build = exported(Reference(
title=_("The CI build that produced this report."),
# Really ICIBuild, patched in _schema_circular_imports.py.
- schema=Interface, required=False, readonly=True)
+ schema=Interface, required=False, readonly=True))
@mutator_for(result)
@operation_parameters(result=copy_field(result))
@@ -260,6 +260,9 @@ class IRevisionStatusArtifactSet(Interface):
def findByReport(report):
"""Returns the set of artifacts for a given report."""
+ def findByCIBuild(ci_build):
+ """Return all `RevisionStatusArtifact`s for a CI build."""
+
def getByRepositoryAndID(repository, id):
"""Returns the artifact for a given repository and ID."""
@@ -270,6 +273,8 @@ class IRevisionStatusArtifact(Interface):
report = Attribute(
"The `RevisionStatusReport` that this artifact is linked to.")
+ library_file_id = Int(
+ title=_("LibraryFileAlias ID"), required=True, readonly=True)
library_file = Attribute(
"The `LibraryFileAlias` object containing information for "
"a revision status report.")
diff --git a/lib/lp/code/interfaces/webservice.py b/lib/lp/code/interfaces/webservice.py
index bfe20b3..a51ce3d 100644
--- a/lib/lp/code/interfaces/webservice.py
+++ b/lib/lp/code/interfaces/webservice.py
@@ -21,6 +21,7 @@ __all__ = [
'IBranchMergeProposal',
'IBranchSet',
'IBranchSubscription',
+ 'ICIBuild',
'ICodeImport',
'ICodeReviewComment',
'ICodeReviewVoteReference',
@@ -56,6 +57,7 @@ from lp.code.interfaces.branch import (
)
from lp.code.interfaces.branchmergeproposal import IBranchMergeProposal
from lp.code.interfaces.branchsubscription import IBranchSubscription
+from lp.code.interfaces.cibuild import ICIBuild
from lp.code.interfaces.codeimport import ICodeImport
from lp.code.interfaces.codereviewcomment import ICodeReviewComment
from lp.code.interfaces.codereviewvote import ICodeReviewVoteReference
diff --git a/lib/lp/code/model/cibuild.py b/lib/lp/code/model/cibuild.py
index f2b44a0..d789a95 100644
--- a/lib/lp/code/model/cibuild.py
+++ b/lib/lp/code/model/cibuild.py
@@ -8,6 +8,7 @@ __all__ = [
]
from datetime import timedelta
+from operator import attrgetter
from lazr.lifecycle.event import ObjectCreatedEvent
import pytz
@@ -53,7 +54,10 @@ from lp.code.interfaces.cibuild import (
)
from lp.code.interfaces.githosting import IGitHostingClient
from lp.code.interfaces.gitrepository import IGitRepository
-from lp.code.interfaces.revisionstatus import IRevisionStatusReportSet
+from lp.code.interfaces.revisionstatus import (
+ IRevisionStatusArtifactSet,
+ IRevisionStatusReportSet,
+ )
from lp.code.model.gitref import GitRef
from lp.code.model.lpcraft import load_configuration
from lp.registry.interfaces.pocket import PackagePublishingPocket
@@ -443,6 +447,15 @@ class CIBuild(PackageBuildMixin, StormBase):
raise NotFoundError(filename)
+ def getFileUrls(self):
+ artifacts = getUtility(IRevisionStatusArtifactSet).findByCIBuild(self)
+ load_related(LibraryFileAlias, artifacts, ["library_file_id"])
+ artifacts = sorted(
+ artifacts, key=attrgetter("library_file.filename", "id"))
+ return [
+ ProxiedLibraryFileAlias(artifact.library_file, artifact).http_url
+ for artifact in artifacts]
+
def verifySuccessfulUpload(self) -> bool:
"""See `IPackageBuild`."""
# We have no interesting checks to perform here.
diff --git a/lib/lp/code/model/revisionstatus.py b/lib/lp/code/model/revisionstatus.py
index 78fc85e..87edccc 100644
--- a/lib/lp/code/model/revisionstatus.py
+++ b/lib/lp/code/model/revisionstatus.py
@@ -284,6 +284,13 @@ class RevisionStatusArtifactSet:
RevisionStatusArtifact,
RevisionStatusArtifact.report == report)
+ def findByCIBuild(self, ci_build):
+ """See `IRevisionStatusArtifactSet`."""
+ return IStore(RevisionStatusArtifact).find(
+ RevisionStatusArtifact,
+ RevisionStatusArtifact.report == RevisionStatusReport.id,
+ RevisionStatusReport.ci_build == ci_build)
+
def getByRepositoryAndID(self, repository, id):
return IStore(RevisionStatusArtifact).find(
RevisionStatusArtifact,
diff --git a/lib/lp/code/model/tests/test_cibuild.py b/lib/lp/code/model/tests/test_cibuild.py
index 80558cf..899b09b 100644
--- a/lib/lp/code/model/tests/test_cibuild.py
+++ b/lib/lp/code/model/tests/test_cibuild.py
@@ -10,13 +10,16 @@ from datetime import (
import hashlib
from textwrap import dedent
from unittest.mock import Mock
+from urllib.request import urlopen
from fixtures import MockPatchObject
from pymacaroons import Macaroon
import pytz
from storm.locals import Store
from testtools.matchers import (
+ ContainsDict,
Equals,
+ Is,
MatchesListwise,
MatchesSetwise,
MatchesStructure,
@@ -61,17 +64,27 @@ from lp.code.tests.helpers import GitHostingFixture
from lp.registry.interfaces.series import SeriesStatus
from lp.services.authserver.xmlrpc import AuthServerAPIView
from lp.services.config import config
+from lp.services.librarian.browser import ProxiedLibraryFileAlias
from lp.services.log.logger import BufferLogger
from lp.services.macaroons.interfaces import IMacaroonIssuer
from lp.services.macaroons.testing import MacaroonTestMixin
from lp.services.propertycache import clear_property_cache
+from lp.services.webapp.interfaces import OAuthPermission
from lp.testing import (
+ ANONYMOUS,
+ api_url,
+ login,
+ logout,
person_logged_in,
StormStatementRecorder,
TestCaseWithFactory,
)
-from lp.testing.layers import LaunchpadZopelessLayer
+from lp.testing.layers import (
+ LaunchpadFunctionalLayer,
+ LaunchpadZopelessLayer,
+ )
from lp.testing.matchers import HasQueryCount
+from lp.testing.pages import webservice_for_person
from lp.xmlrpc.interfaces import IPrivateApplication
@@ -1006,6 +1019,159 @@ class TestDetermineDASesToBuild(TestCaseWithFactory):
)
+class TestCIBuildWebservice(TestCaseWithFactory):
+
+ layer = LaunchpadFunctionalLayer
+
+ def setUp(self):
+ super().setUp()
+ self.person = self.factory.makePerson()
+ self.webservice = webservice_for_person(
+ self.person, permission=OAuthPermission.WRITE_PRIVATE)
+ self.webservice.default_api_version = "devel"
+ login(ANONYMOUS)
+
+ def getURL(self, obj):
+ return self.webservice.getAbsoluteUrl(api_url(obj))
+
+ def test_properties(self):
+ # The basic properties of a CI build are sensible.
+ db_build = self.factory.makeCIBuild()
+ build_url = api_url(db_build)
+ logout()
+ build = self.webservice.get(build_url).jsonBody()
+ with person_logged_in(self.person):
+ self.assertThat(build, ContainsDict({
+ "git_repository_link": Equals(
+ self.getURL(db_build.git_repository)),
+ "commit_sha1": Equals(db_build.commit_sha1),
+ "distro_arch_series_link": Equals(
+ self.getURL(db_build.distro_arch_series)),
+ "arch_tag": Equals(
+ db_build.distro_arch_series.architecturetag),
+ "score": Is(None),
+ "stages": Equals([[["test", 0]]]),
+ "results": Equals({}),
+ "can_be_rescored": Is(False),
+ "can_be_retried": Is(False),
+ "can_be_cancelled": Is(False),
+ }))
+
+ def test_public(self):
+ # A CI build with a public repository is itself public.
+ db_build = self.factory.makeCIBuild()
+ build_url = api_url(db_build)
+ unpriv_webservice = webservice_for_person(
+ self.factory.makePerson(), permission=OAuthPermission.WRITE_PUBLIC)
+ unpriv_webservice.default_api_version = "devel"
+ logout()
+ self.assertEqual(200, self.webservice.get(build_url).status)
+ self.assertEqual(200, unpriv_webservice.get(build_url).status)
+
+ def test_private(self):
+ # A CI build with a private repository is private.
+ db_repository = self.factory.makeGitRepository(
+ owner=self.person, information_type=InformationType.USERDATA)
+ with person_logged_in(self.person):
+ db_build = self.factory.makeCIBuild(git_repository=db_repository)
+ build_url = api_url(db_build)
+ unpriv_webservice = webservice_for_person(
+ self.factory.makePerson(), permission=OAuthPermission.WRITE_PUBLIC)
+ unpriv_webservice.default_api_version = "devel"
+ logout()
+ self.assertEqual(200, self.webservice.get(build_url).status)
+ self.assertEqual(401, unpriv_webservice.get(build_url).status)
+
+ def test_cancel(self):
+ # The owner of a build's repository can cancel the build.
+ db_repository = self.factory.makeGitRepository(owner=self.person)
+ db_build = self.factory.makeCIBuild(git_repository=db_repository)
+ db_build.queueBuild()
+ build_url = api_url(db_build)
+ unpriv_webservice = webservice_for_person(
+ self.factory.makePerson(), permission=OAuthPermission.WRITE_PUBLIC)
+ unpriv_webservice.default_api_version = "devel"
+ logout()
+ build = self.webservice.get(build_url).jsonBody()
+ self.assertTrue(build["can_be_cancelled"])
+ response = unpriv_webservice.named_post(build["self_link"], "cancel")
+ self.assertEqual(401, response.status)
+ response = self.webservice.named_post(build["self_link"], "cancel")
+ self.assertEqual(200, response.status)
+ build = self.webservice.get(build_url).jsonBody()
+ self.assertFalse(build["can_be_cancelled"])
+ with person_logged_in(self.person):
+ self.assertEqual(BuildStatus.CANCELLED, db_build.status)
+
+ def test_rescore(self):
+ # Buildd administrators can rescore builds.
+ db_build = self.factory.makeCIBuild()
+ db_build.queueBuild()
+ build_url = api_url(db_build)
+ buildd_admin = self.factory.makePerson(
+ member_of=[getUtility(ILaunchpadCelebrities).buildd_admin])
+ buildd_admin_webservice = webservice_for_person(
+ buildd_admin, permission=OAuthPermission.WRITE_PUBLIC)
+ buildd_admin_webservice.default_api_version = "devel"
+ logout()
+ build = self.webservice.get(build_url).jsonBody()
+ self.assertEqual(2600, build["score"])
+ self.assertTrue(build["can_be_rescored"])
+ response = self.webservice.named_post(
+ build["self_link"], "rescore", score=5000)
+ self.assertEqual(401, response.status)
+ response = buildd_admin_webservice.named_post(
+ build["self_link"], "rescore", score=5000)
+ self.assertEqual(200, response.status)
+ build = self.webservice.get(build_url).jsonBody()
+ self.assertEqual(5000, build["score"])
+
+ def assertCanOpenRedirectedUrl(self, browser, url):
+ browser.open(url)
+ self.assertEqual(303, int(browser.headers["Status"].split(" ", 1)[0]))
+ urlopen(browser.headers["Location"]).close()
+
+ def test_logs(self):
+ # API clients can fetch the build and upload logs.
+ db_build = self.factory.makeCIBuild()
+ db_build.setLog(self.factory.makeLibraryFileAlias("buildlog.txt.gz"))
+ db_build.storeUploadLog("uploaded")
+ build_url = api_url(db_build)
+ logout()
+ build = self.webservice.get(build_url).jsonBody()
+ browser = self.getNonRedirectingBrowser(user=self.person)
+ browser.raiseHttpErrors = False
+ self.assertIsNotNone(build["build_log_url"])
+ self.assertCanOpenRedirectedUrl(browser, build["build_log_url"])
+ self.assertIsNotNone(build["upload_log_url"])
+ self.assertCanOpenRedirectedUrl(browser, build["upload_log_url"])
+
+ def test_getFileUrls(self):
+ # API clients can fetch files attached to builds.
+ db_build = self.factory.makeCIBuild()
+ db_reports = [
+ self.factory.makeRevisionStatusReport(ci_build=db_build)
+ for _ in range(2)]
+ db_artifacts = []
+ for db_report in db_reports:
+ for _ in range(2):
+ db_artifacts.append(self.factory.makeRevisionStatusArtifact(
+ lfa=self.factory.makeLibraryFileAlias(), report=db_report))
+ build_url = api_url(db_build)
+ file_urls = [
+ ProxiedLibraryFileAlias(
+ db_artifact.library_file, db_artifact).http_url
+ for db_artifact in db_artifacts]
+ logout()
+ response = self.webservice.named_get(build_url, "getFileUrls")
+ self.assertEqual(200, response.status)
+ self.assertContentEqual(file_urls, response.jsonBody())
+ browser = self.getNonRedirectingBrowser(user=self.person)
+ browser.raiseHttpErrors = False
+ for file_url in file_urls:
+ self.assertCanOpenRedirectedUrl(browser, file_url)
+
+
class TestCIBuildMacaroonIssuer(MacaroonTestMixin, TestCaseWithFactory):
"""Test CIBuild macaroon issuing and verification."""
