launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #28029
[Merge] ~cjwatson/launchpad:ci-build-behaviour into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:ci-build-behaviour into launchpad:master with ~cjwatson/launchpad:ci-build as a prerequisite.
Commit message:
Add a build behaviour for CI builds
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/414769
There's no way to request these builds, and no uploader handling to process the output of them either; and I haven't implemented incremental fetching of the logs/output of individual jobs during the build. Still, this makes a bit more progress on putting the pieces together.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:ci-build-behaviour into launchpad:master.
diff --git a/database/schema/security.cfg b/database/schema/security.cfg
index a6f6ca4..2d20f6c 100644
--- a/database/schema/security.cfg
+++ b/database/schema/security.cfg
@@ -1018,6 +1018,7 @@ public.charmrecipe = SELECT
public.charmrecipebuild = SELECT, UPDATE
public.charmrecipebuildjob = SELECT, INSERT
public.charmrecipejob = SELECT
+public.cibuild = SELECT, UPDATE
public.component = SELECT
public.componentselection = SELECT
public.distribution = SELECT, UPDATE
diff --git a/lib/lp/buildmaster/model/buildfarmjobbehaviour.py b/lib/lp/buildmaster/model/buildfarmjobbehaviour.py
index a7537d3..1792d78 100644
--- a/lib/lp/buildmaster/model/buildfarmjobbehaviour.py
+++ b/lib/lp/buildmaster/model/buildfarmjobbehaviour.py
@@ -341,7 +341,8 @@ class BuildFarmJobBehaviourBase:
transaction.commit()
@defer.inlineCallbacks
- def _downloadFiles(self, filemap, upload_path, logger):
+ def _downloadFiles(self, worker_status, upload_path, logger):
+ filemap = worker_status["filemap"]
filenames_to_download = []
for filename, sha1 in filemap.items():
logger.info("Grabbing file: %s (%s)" % (
@@ -365,7 +366,6 @@ class BuildFarmJobBehaviourBase:
uploader.
"""
build = self.build
- filemap = worker_status['filemap']
# If this is a binary package build, discard it if its source is
# no longer published.
@@ -393,7 +393,7 @@ class BuildFarmJobBehaviourBase:
grab_dir, str(build.archive.id), build.distribution.name)
os.makedirs(upload_path)
- yield self._downloadFiles(filemap, upload_path, logger)
+ yield self._downloadFiles(worker_status, upload_path, logger)
transaction.commit()
diff --git a/lib/lp/code/configure.zcml b/lib/lp/code/configure.zcml
index 6bbd418..dc3d63e 100644
--- a/lib/lp/code/configure.zcml
+++ b/lib/lp/code/configure.zcml
@@ -1303,6 +1303,13 @@
<allow interface="lp.buildmaster.interfaces.buildfarmjob.ISpecificBuildFarmJobSource" />
</securedutility>
+ <!-- CIBuildBehaviour -->
+ <adapter
+ for="lp.code.interfaces.cibuild.ICIBuild"
+ provides="lp.buildmaster.interfaces.buildfarmjobbehaviour.IBuildFarmJobBehaviour"
+ factory="lp.code.model.cibuildbehaviour.CIBuildBehaviour"
+ permission="zope.Public" />
+
<webservice:register module="lp.code.interfaces.webservice" />
<adapter
diff --git a/lib/lp/code/interfaces/cibuild.py b/lib/lp/code/interfaces/cibuild.py
index 96781f1..4537b1c 100644
--- a/lib/lp/code/interfaces/cibuild.py
+++ b/lib/lp/code/interfaces/cibuild.py
@@ -4,8 +4,11 @@
"""Interfaces for CI builds."""
__all__ = [
+ "CannotFetchConfiguration",
+ "CannotParseConfiguration",
"ICIBuild",
"ICIBuildSet",
+ "MissingConfiguration",
]
from lazr.restful.fields import Reference
@@ -31,6 +34,25 @@ from lp.services.database.constants import DEFAULT
from lp.soyuz.interfaces.distroarchseries import IDistroArchSeries
+class MissingConfiguration(Exception):
+ """The repository for this CI build does not have a .launchpad.yaml."""
+
+ def __init__(self, name):
+ super().__init__("Cannot find .launchpad.yaml in %s" % name)
+
+
+class CannotFetchConfiguration(Exception):
+ """Launchpad cannot fetch this CI build's .launchpad.yaml."""
+
+ def __init__(self, message, unsupported_remote=False):
+ super().__init__(message)
+ self.unsupported_remote = unsupported_remote
+
+
+class CannotParseConfiguration(Exception):
+ """Launchpad cannot parse this CI build's .launchpad.yaml."""
+
+
class ICIBuildView(IPackageBuildView):
"""`ICIBuild` attributes that require launchpad.View."""
@@ -68,6 +90,21 @@ class ICIBuildView(IPackageBuildView):
"The date when the build completed or is estimated to complete."),
readonly=True)
+ def getConfiguration(logger=None):
+ """Fetch a CI build's .launchpad.yaml from code hosting, if possible.
+
+ :param logger: An optional logger.
+
+ :return: The build's parsed .launchpad.yaml.
+ :raises MissingConfiguration: if this package has no
+ .launchpad.yaml.
+ :raises CannotFetchConfiguration: if it was not possible to fetch
+ .launchpad.yaml from the code hosting backend for some other
+ reason.
+ :raises CannotParseConfiguration: if the fetched .launchpad.yaml
+ cannot be parsed.
+ """
+
class ICIBuildEdit(IBuildFarmJobEdit):
"""`ICIBuild` methods that require launchpad.Edit."""
diff --git a/lib/lp/code/model/cibuild.py b/lib/lp/code/model/cibuild.py
index 12f2633..f9c9031 100644
--- a/lib/lp/code/model/cibuild.py
+++ b/lib/lp/code/model/cibuild.py
@@ -31,10 +31,18 @@ from lp.buildmaster.enums import (
from lp.buildmaster.interfaces.buildfarmjob import IBuildFarmJobSource
from lp.buildmaster.model.buildfarmjob import SpecificBuildFarmJobSourceMixin
from lp.buildmaster.model.packagebuild import PackageBuildMixin
+from lp.code.errors import (
+ GitRepositoryBlobNotFound,
+ GitRepositoryScanFault,
+ )
from lp.code.interfaces.cibuild import (
+ CannotFetchConfiguration,
+ CannotParseConfiguration,
ICIBuild,
ICIBuildSet,
+ MissingConfiguration,
)
+from lp.code.model.lpcraft import load_configuration
from lp.registry.interfaces.pocket import PackagePublishingPocket
from lp.registry.interfaces.series import SeriesStatus
from lp.registry.model.distribution import Distribution
@@ -57,6 +65,16 @@ from lp.services.propertycache import cachedproperty
from lp.soyuz.model.distroarchseries import DistroArchSeries
+def parse_configuration(git_repository, blob):
+ try:
+ return load_configuration(blob)
+ except Exception as e:
+ # Don't bother logging parsing errors from user-supplied YAML.
+ raise CannotParseConfiguration(
+ "Cannot parse .launchpad.yaml from %s: %s" %
+ (git_repository.unique_name, e))
+
+
@implementer(ICIBuild)
class CIBuild(PackageBuildMixin, StormBase):
"""See `ICIBuild`."""
@@ -258,6 +276,33 @@ class CIBuild(PackageBuildMixin, StormBase):
return self.eta
return self.date_finished
+ def getConfiguration(self, logger=None):
+ """See `ICIBuild`."""
+ try:
+ paths = (
+ ".launchpad.yaml",
+ )
+ for path in paths:
+ try:
+ blob = self.git_repository.getBlob(
+ path, rev=self.commit_sha1)
+ break
+ except GitRepositoryBlobNotFound:
+ pass
+ else:
+ if logger is not None:
+ logger.exception(
+ "Cannot find .launchpad.yaml in %s" %
+ self.git_repository.unique_name)
+ raise MissingConfiguration(self.git_repository.unique_name)
+ except GitRepositoryScanFault as e:
+ msg = "Failed to get .launchpad.yaml from %s"
+ if logger is not None:
+ logger.exception(msg, self.git_repository.unique_name)
+ raise CannotFetchConfiguration(
+ "%s: %s" % (msg % self.git_repository.unique_name, e))
+ return parse_configuration(self.git_repository, blob)
+
def verifySuccessfulUpload(self):
"""See `IPackageBuild`."""
# We have no interesting checks to perform here.
diff --git a/lib/lp/code/model/cibuildbehaviour.py b/lib/lp/code/model/cibuildbehaviour.py
new file mode 100644
index 0000000..77f04af
--- /dev/null
+++ b/lib/lp/code/model/cibuildbehaviour.py
@@ -0,0 +1,113 @@
+# Copyright 2022 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""An `IBuildFarmJobBehaviour` for `CIBuild`."""
+
+__all__ = [
+ "CIBuildBehaviour",
+ ]
+
+import json
+import os
+
+from twisted.internet import defer
+from zope.component import adapter
+from zope.interface import implementer
+
+from lp.buildmaster.builderproxy import BuilderProxyMixin
+from lp.buildmaster.enums import BuildBaseImageType
+from lp.buildmaster.interfaces.builder import CannotBuild
+from lp.buildmaster.interfaces.buildfarmjobbehaviour import (
+ IBuildFarmJobBehaviour,
+ )
+from lp.buildmaster.model.buildfarmjobbehaviour import (
+ BuildFarmJobBehaviourBase,
+ )
+from lp.code.interfaces.cibuild import ICIBuild
+from lp.soyuz.adapters.archivedependencies import (
+ get_sources_list_for_building,
+ )
+
+
+@adapter(ICIBuild)
+@implementer(IBuildFarmJobBehaviour)
+class CIBuildBehaviour(BuilderProxyMixin, BuildFarmJobBehaviourBase):
+ """Dispatches `CIBuild` jobs to builders."""
+
+ builder_type = "ci"
+ image_types = [BuildBaseImageType.LXD, BuildBaseImageType.CHROOT]
+
+ ALLOWED_STATUS_NOTIFICATIONS = []
+
+ def getLogFileName(self):
+ return "buildlog_ci_%s_%s_%s.txt" % (
+ self.build.git_repository.name, self.build.commit_sha1,
+ self.build.status.name)
+
+ def verifyBuildRequest(self, logger):
+ """Assert some pre-build checks.
+
+ The build request is checked:
+ * Virtualized builds can't build on a non-virtual builder
+ * We have a base image
+ """
+ build = self.build
+ if build.virtualized and not self._builder.virtualized:
+ raise AssertionError(
+ "Attempt to build virtual item on a non-virtual builder.")
+
+ chroot = build.distro_arch_series.getChroot(pocket=build.pocket)
+ if chroot is None:
+ raise CannotBuild(
+ "Missing chroot for %s" % build.distro_arch_series.displayname)
+
+ @defer.inlineCallbacks
+ def extraBuildArgs(self, logger=None):
+ """Return extra builder arguments for this build."""
+ build = self.build
+ try:
+ configuration = self.build.getConfiguration(logger=logger)
+ except Exception as e:
+ raise CannotBuild(str(e))
+ stages = []
+ if not configuration.pipeline:
+ raise CannotBuild(
+ "No jobs defined for %s:%s" %
+ (build.git_repository.unique_name, build.commit_sha1))
+ for stage in configuration.pipeline:
+ jobs = []
+ for job_name in stage:
+ if job_name not in configuration.jobs:
+ raise CannotBuild(
+ "Job '%s' in pipeline for %s:%s but not in jobs" %
+ (job_name,
+ build.git_repository.unique_name, build.commit_sha1))
+ for i in range(len(configuration.jobs[job_name])):
+ jobs.append((job_name, i))
+ stages.append(jobs)
+
+ args = yield super().extraBuildArgs(logger=logger)
+ yield self.addProxyArgs(args)
+ args["archives"], args["trusted_keys"] = (
+ yield get_sources_list_for_building(
+ self, build.distro_arch_series, None, logger=logger))
+ args["jobs"] = stages
+ args["git_repository"] = build.git_repository.git_https_url
+ args["commit_sha1"] = build.commit_sha1
+ args["private"] = build.is_private
+ return args
+
+ def verifySuccessfulBuild(self):
+ """See `IBuildFarmJobBehaviour`."""
+ # We have no interesting checks to perform here.
+
+ @defer.inlineCallbacks
+ def _downloadFiles(self, worker_status, upload_path, logger):
+ # In addition to downloading everything from the filemap, we need to
+ # save the "jobs" field in order to reliably map files to individual
+ # CI jobs.
+ if "jobs" in worker_status:
+ jobs_path = os.path.join(upload_path, "jobs.json")
+ with open(jobs_path, "w") as jobs_file:
+ json.dump(worker_status["jobs"], jobs_file)
+ yield super()._downloadFiles(worker_status, upload_path, logger)
diff --git a/lib/lp/code/model/tests/test_cibuild.py b/lib/lp/code/model/tests/test_cibuild.py
index 784a241..133bed2 100644
--- a/lib/lp/code/model/tests/test_cibuild.py
+++ b/lib/lp/code/model/tests/test_cibuild.py
@@ -7,9 +7,13 @@ from datetime import (
datetime,
timedelta,
)
+from textwrap import dedent
import pytz
-from testtools.matchers import Equals
+from testtools.matchers import (
+ Equals,
+ MatchesStructure,
+ )
from zope.component import getUtility
from zope.security.proxy import removeSecurityProxy
@@ -17,10 +21,18 @@ from lp.app.enums import InformationType
from lp.buildmaster.enums import BuildStatus
from lp.buildmaster.interfaces.buildqueue import IBuildQueue
from lp.buildmaster.interfaces.packagebuild import IPackageBuild
+from lp.code.errors import (
+ GitRepositoryBlobNotFound,
+ GitRepositoryScanFault,
+ )
from lp.code.interfaces.cibuild import (
+ CannotFetchConfiguration,
+ CannotParseConfiguration,
ICIBuild,
ICIBuildSet,
+ MissingConfiguration,
)
+from lp.code.tests.helpers import GitHostingFixture
from lp.registry.interfaces.series import SeriesStatus
from lp.services.propertycache import clear_property_cache
from lp.testing import (
@@ -222,6 +234,55 @@ class TestCIBuild(TestCaseWithFactory):
clear_property_cache(build)
self.assertFalse(build.estimate)
+ def test_getConfiguration(self):
+ build = self.factory.makeCIBuild()
+ das = build.distro_arch_series
+ self.useFixture(GitHostingFixture(blob=dedent("""\
+ pipeline: [test]
+ jobs:
+ test:
+ series: {}
+ architectures: [{}]
+ """.format(das.distroseries.name, das.architecturetag)).encode()))
+ self.assertThat(
+ build.getConfiguration(),
+ MatchesStructure.byEquality(
+ pipeline=[["test"]],
+ jobs={
+ "test": [{
+ "series": das.distroseries.name,
+ "architectures": [das.architecturetag],
+ }]}))
+
+ def test_getConfiguration_not_found(self):
+ build = self.factory.makeCIBuild()
+ self.useFixture(GitHostingFixture()).getBlob.failure = (
+ GitRepositoryBlobNotFound(
+ build.git_repository.getInternalPath(), ".launchpad.yaml",
+ rev=build.commit_sha1))
+ self.assertRaisesWithContent(
+ MissingConfiguration,
+ "Cannot find .launchpad.yaml in %s" % (
+ build.git_repository.unique_name),
+ build.getConfiguration)
+
+ def test_getConfiguration_fetch_error(self):
+ build = self.factory.makeCIBuild()
+ self.useFixture(GitHostingFixture()).getBlob.failure = (
+ GitRepositoryScanFault("Boom"))
+ self.assertRaisesWithContent(
+ CannotFetchConfiguration,
+ "Failed to get .launchpad.yaml from %s: Boom" % (
+ build.git_repository.unique_name),
+ build.getConfiguration)
+
+ def test_getConfiguration_invalid_data(self):
+ build = self.factory.makeCIBuild()
+ hosting_fixture = self.useFixture(GitHostingFixture())
+ for invalid_result in (None, 123, b"", b"[][]", b"#name:test", b"]"):
+ hosting_fixture.getBlob.result = invalid_result
+ self.assertRaises(CannotParseConfiguration, build.getConfiguration)
+
class TestCIBuildSet(TestCaseWithFactory):
diff --git a/lib/lp/code/model/tests/test_cibuildbehaviour.py b/lib/lp/code/model/tests/test_cibuildbehaviour.py
new file mode 100644
index 0000000..808144a
--- /dev/null
+++ b/lib/lp/code/model/tests/test_cibuildbehaviour.py
@@ -0,0 +1,474 @@
+# Copyright 2022 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Test CI build behaviour."""
+
+import base64
+from datetime import datetime
+import json
+import os.path
+import re
+from textwrap import dedent
+import time
+from urllib.parse import urlsplit
+import uuid
+
+from fixtures import MockPatch
+from testtools import ExpectedException
+from testtools.matchers import (
+ ContainsDict,
+ Equals,
+ Is,
+ IsInstance,
+ MatchesDict,
+ MatchesListwise,
+ StartsWith,
+ )
+from testtools.twistedsupport import (
+ AsynchronousDeferredRunTestForBrokenTwisted,
+ )
+from twisted.internet import defer
+from zope.component import getUtility
+
+from lp.app.enums import InformationType
+from lp.app.interfaces.launchpad import ILaunchpadCelebrities
+from lp.archivepublisher.interfaces.archivegpgsigningkey import (
+ IArchiveGPGSigningKey,
+ )
+from lp.buildmaster.enums import (
+ BuildBaseImageType,
+ BuildStatus,
+ )
+from lp.buildmaster.interactor import shut_down_default_process_pool
+from lp.buildmaster.interfaces.builder import CannotBuild
+from lp.buildmaster.interfaces.buildfarmjobbehaviour import (
+ IBuildFarmJobBehaviour,
+ )
+from lp.buildmaster.interfaces.processor import IProcessorSet
+from lp.buildmaster.tests.builderproxy import (
+ InProcessProxyAuthAPIFixture,
+ ProxyURLMatcher,
+ RevocationEndpointMatcher,
+ )
+from lp.buildmaster.tests.mock_workers import (
+ MockBuilder,
+ OkWorker,
+ WorkerTestHelpers,
+ )
+from lp.buildmaster.tests.test_buildfarmjobbehaviour import (
+ TestGetUploadMethodsMixin,
+ TestHandleStatusMixin,
+ TestVerifySuccessfulBuildMixin,
+ )
+from lp.code.errors import GitRepositoryBlobNotFound
+from lp.code.model.cibuildbehaviour import CIBuildBehaviour
+from lp.code.tests.helpers import GitHostingFixture
+from lp.services.config import config
+from lp.services.log.logger import (
+ BufferLogger,
+ DevNullLogger,
+ )
+from lp.services.statsd.tests import StatsMixin
+from lp.services.timeout import (
+ get_default_timeout_function,
+ set_default_timeout_function,
+ )
+from lp.services.webapp import canonical_url
+from lp.soyuz.adapters.archivedependencies import (
+ get_sources_list_for_building,
+ )
+from lp.soyuz.enums import PackagePublishingStatus
+from lp.soyuz.tests.soyuz import Base64KeyMatches
+from lp.testing import TestCaseWithFactory
+from lp.testing.dbuser import dbuser
+from lp.testing.gpgkeys import gpgkeysdir
+from lp.testing.keyserver import InProcessKeyServerFixture
+from lp.testing.layers import ZopelessDatabaseLayer
+
+
+class TestCIBuildBehaviourBase(TestCaseWithFactory):
+
+ layer = ZopelessDatabaseLayer
+
+ def makeJob(self, **kwargs):
+ ubuntu = getUtility(ILaunchpadCelebrities).ubuntu
+ distroseries = self.factory.makeDistroSeries(distribution=ubuntu)
+ processor = getUtility(IProcessorSet).getByName("386")
+ distroarchseries = self.factory.makeDistroArchSeries(
+ distroseries=distroseries, architecturetag="i386",
+ processor=processor)
+
+ # Taken from test_archivedependencies.py
+ for component_name in ("main", "universe"):
+ self.factory.makeComponentSelection(distroseries, component_name)
+
+ build = self.factory.makeCIBuild(
+ distro_arch_series=distroarchseries, **kwargs)
+ return IBuildFarmJobBehaviour(build)
+
+
+class TestCIBuildBehaviour(TestCIBuildBehaviourBase):
+
+ def test_provides_interface(self):
+ # CIBuildBehaviour provides IBuildFarmJobBehaviour.
+ job = CIBuildBehaviour(None)
+ self.assertProvides(job, IBuildFarmJobBehaviour)
+
+ def test_adapts_ICIBuild(self):
+ # IBuildFarmJobBehaviour adapts an ICIBuild.
+ build = self.factory.makeCIBuild()
+ job = IBuildFarmJobBehaviour(build)
+ self.assertProvides(job, IBuildFarmJobBehaviour)
+
+ def test_verifyBuildRequest_valid(self):
+ # verifyBuildRequest doesn't raise any exceptions when called with a
+ # valid builder set.
+ job = self.makeJob()
+ lfa = self.factory.makeLibraryFileAlias(db_only=True)
+ job.build.distro_arch_series.addOrUpdateChroot(lfa)
+ builder = MockBuilder()
+ job.setBuilder(builder, OkWorker())
+ logger = BufferLogger()
+ job.verifyBuildRequest(logger)
+ self.assertEqual("", logger.getLogBuffer())
+
+ def test_verifyBuildRequest_virtual_mismatch(self):
+ # verifyBuildRequest raises on an attempt to build a virtualized
+ # build on a non-virtual builder.
+ job = self.makeJob()
+ lfa = self.factory.makeLibraryFileAlias(db_only=True)
+ job.build.distro_arch_series.addOrUpdateChroot(lfa)
+ builder = MockBuilder(virtualized=False)
+ job.setBuilder(builder, OkWorker())
+ logger = BufferLogger()
+ e = self.assertRaises(AssertionError, job.verifyBuildRequest, logger)
+ self.assertEqual(
+ "Attempt to build virtual item on a non-virtual builder.", str(e))
+
+ def test_verifyBuildRequest_no_chroot(self):
+ # verifyBuildRequest raises when the DAS has no chroot.
+ job = self.makeJob()
+ builder = MockBuilder()
+ job.setBuilder(builder, OkWorker())
+ logger = BufferLogger()
+ e = self.assertRaises(CannotBuild, job.verifyBuildRequest, logger)
+ self.assertIn("Missing chroot", str(e))
+
+
+_unset = object()
+
+
+class TestAsyncCIBuildBehaviour(StatsMixin, TestCIBuildBehaviourBase):
+
+ run_tests_with = AsynchronousDeferredRunTestForBrokenTwisted.make_factory(
+ timeout=30)
+
+ @defer.inlineCallbacks
+ def setUp(self):
+ super().setUp()
+ build_username = "CIBUILD-1"
+ self.token = {"secret": uuid.uuid4().hex,
+ "username": build_username,
+ "timestamp": datetime.utcnow().isoformat()}
+ self.proxy_url = ("http://{username}:{password}";
+ "@{host}:{port}".format(
+ username=self.token["username"],
+ password=self.token["secret"],
+ host=config.builddmaster.builder_proxy_host,
+ port=config.builddmaster.builder_proxy_port))
+ self.proxy_api = self.useFixture(InProcessProxyAuthAPIFixture())
+ yield self.proxy_api.start()
+ self.now = time.time()
+ self.useFixture(MockPatch("time.time", return_value=self.now))
+ self.addCleanup(shut_down_default_process_pool)
+ self.setUpStats()
+ self.addCleanup(
+ set_default_timeout_function, get_default_timeout_function())
+ set_default_timeout_function(lambda: None)
+
+ def makeJob(self, configuration=_unset, **kwargs):
+ # We need a builder in these tests, in order that requesting a proxy
+ # token can piggyback on its reactor and pool.
+ job = super().makeJob(**kwargs)
+ builder = MockBuilder()
+ builder.processor = job.build.processor
+ worker = self.useFixture(WorkerTestHelpers()).getClientWorker()
+ job.setBuilder(builder, worker)
+ self.addCleanup(worker.pool.closeCachedConnections)
+ if configuration is _unset:
+ # Skeleton configuration defining a single job.
+ configuration = dedent("""\
+ pipeline:
+ - [test]
+ jobs:
+ test:
+ series: {}
+ architectures: [{}]
+ """.format(
+ job.build.distro_arch_series.distroseries.name,
+ job.build.distro_arch_series.architecturetag)).encode()
+ hosting_fixture = self.useFixture(
+ GitHostingFixture(blob=configuration))
+ if configuration is None:
+ hosting_fixture.getBlob.failure = GitRepositoryBlobNotFound(
+ job.build.git_repository.getInternalPath(), ".launchpad.yaml",
+ rev=job.build.commit_sha1)
+ return job
+
+ @defer.inlineCallbacks
+ def test_composeBuildRequest(self):
+ job = self.makeJob()
+ lfa = self.factory.makeLibraryFileAlias(db_only=True)
+ job.build.distro_arch_series.addOrUpdateChroot(lfa)
+ build_request = yield job.composeBuildRequest(None)
+ self.assertThat(build_request, MatchesListwise([
+ Equals("ci"),
+ Equals(job.build.distro_arch_series),
+ Equals(job.build.pocket),
+ Equals({}),
+ IsInstance(dict),
+ ]))
+
+ @defer.inlineCallbacks
+ def test_requestProxyToken_unconfigured(self):
+ self.pushConfig(
+ "builddmaster", builder_proxy_auth_api_admin_secret=None)
+ job = self.makeJob()
+ expected_exception_msg = (
+ "builder_proxy_auth_api_admin_secret is not configured.")
+ with ExpectedException(CannotBuild, expected_exception_msg):
+ yield job.extraBuildArgs()
+
+ @defer.inlineCallbacks
+ def test_requestProxyToken(self):
+ job = self.makeJob()
+ yield job.extraBuildArgs()
+ expected_uri = urlsplit(
+ config.builddmaster.builder_proxy_auth_api_endpoint
+ ).path.encode("UTF-8")
+ self.assertThat(self.proxy_api.tokens.requests, MatchesListwise([
+ MatchesDict({
+ "method": Equals(b"POST"),
+ "uri": Equals(expected_uri),
+ "headers": ContainsDict({
+ b"Authorization": MatchesListwise([
+ Equals(b"Basic " + base64.b64encode(
+ b"admin-launchpad.test:admin-secret"))]),
+ b"Content-Type": MatchesListwise([
+ Equals(b"application/json"),
+ ]),
+ }),
+ "json": MatchesDict({
+ "username": StartsWith(job.build.build_cookie + "-"),
+ }),
+ }),
+ ]))
+
+ @defer.inlineCallbacks
+ def test_extraBuildArgs_git(self):
+ # extraBuildArgs returns appropriate arguments if asked to build a
+ # job for a Git commit.
+ job = self.makeJob()
+ expected_archives, expected_trusted_keys = (
+ yield get_sources_list_for_building(
+ job, job.build.distro_arch_series, None))
+ for archive_line in expected_archives:
+ self.assertIn("universe", archive_line)
+ with dbuser(config.builddmaster.dbuser):
+ args = yield job.extraBuildArgs()
+ self.assertThat(args, MatchesDict({
+ "archive_private": Is(False),
+ "archives": Equals(expected_archives),
+ "arch_tag": Equals("i386"),
+ "build_url": Equals(canonical_url(job.build)),
+ "commit_sha1": Equals(job.build.commit_sha1),
+ "fast_cleanup": Is(True),
+ "git_repository": Equals(job.build.git_repository.git_https_url),
+ "jobs": Equals([[("test", 0)]]),
+ "private": Is(False),
+ "proxy_url": ProxyURLMatcher(job, self.now),
+ "revocation_endpoint": RevocationEndpointMatcher(job, self.now),
+ "series": Equals(job.build.distro_series.name),
+ "trusted_keys": Equals(expected_trusted_keys),
+ }))
+
+ @defer.inlineCallbacks
+ def test_extraBuildArgs_archive_trusted_keys(self):
+ # If the archive has a signing key, extraBuildArgs sends it.
+ yield self.useFixture(InProcessKeyServerFixture()).start()
+ job = self.makeJob()
+ distribution = job.build.distribution
+ key_path = os.path.join(gpgkeysdir, "ppa-sample@xxxxxxxxxxxxxxxxx")
+ yield IArchiveGPGSigningKey(distribution.main_archive).setSigningKey(
+ key_path, async_keyserver=True)
+ self.factory.makeBinaryPackagePublishingHistory(
+ distroarchseries=job.build.distro_arch_series,
+ pocket=job.build.pocket, archive=distribution.main_archive,
+ status=PackagePublishingStatus.PUBLISHED)
+ with dbuser(config.builddmaster.dbuser):
+ args = yield job.extraBuildArgs()
+ self.assertThat(args["trusted_keys"], MatchesListwise([
+ Base64KeyMatches("0D57E99656BEFB0897606EE9A022DD1F5001B46D"),
+ ]))
+
+ @defer.inlineCallbacks
+ def test_extraBuildArgs_archives_primary(self):
+ # The build uses the release, security, and updates pockets from the
+ # primary archive.
+ job = self.makeJob()
+ expected_archives = [
+ "deb %s %s main universe" % (
+ job.archive.archive_url, job.build.distro_series.name),
+ "deb %s %s-security main universe" % (
+ job.archive.archive_url, job.build.distro_series.name),
+ "deb %s %s-updates main universe" % (
+ job.archive.archive_url, job.build.distro_series.name),
+ ]
+ with dbuser(config.builddmaster.dbuser):
+ extra_args = yield job.extraBuildArgs()
+ self.assertEqual(expected_archives, extra_args["archives"])
+
+ @defer.inlineCallbacks
+ def test_extraBuildArgs_private(self):
+ # If the repository is private, extraBuildArgs sends the appropriate
+ # arguments.
+ repository = self.factory.makeGitRepository(
+ information_type=InformationType.USERDATA)
+ job = self.makeJob(git_repository=repository)
+ with dbuser(config.builddmaster.dbuser):
+ args = yield job.extraBuildArgs()
+ self.assertTrue(args["private"])
+
+ @defer.inlineCallbacks
+ def test_composeBuildRequest_proxy_url_set(self):
+ job = self.makeJob()
+ build_request = yield job.composeBuildRequest(None)
+ self.assertThat(
+ build_request[4]["proxy_url"], ProxyURLMatcher(job, self.now))
+
+ @defer.inlineCallbacks
+ def test_composeBuildRequest_unparseable(self):
+ # If the job's configuration file fails to parse,
+ # composeBuildRequest raises CannotBuild.
+ job = self.makeJob(configuration=b"")
+ expected_exception_msg = (
+ r"Cannot parse \.launchpad\.yaml from .*: "
+ r"Empty configuration file")
+ with ExpectedException(CannotBuild, expected_exception_msg):
+ yield job.composeBuildRequest(None)
+
+ @defer.inlineCallbacks
+ def test_composeBuildRequest_no_jobs_defined(self):
+ # If the job's configuration does not define any jobs,
+ # composeBuildRequest raises CannotBuild.
+ job = self.makeJob(configuration=b"pipeline: []\njobs: {}\n")
+ expected_exception_msg = re.escape(
+ "No jobs defined for %s:%s" % (
+ job.build.git_repository.unique_name, job.build.commit_sha1))
+ with ExpectedException(CannotBuild, expected_exception_msg):
+ yield job.composeBuildRequest(None)
+
+ @defer.inlineCallbacks
+ def test_composeBuildRequest_undefined_job(self):
+ # If the job's configuration has a pipeline that defines a job not
+ # in the jobs matrix, composeBuildRequest raises CannotBuild.
+ job = self.makeJob(configuration=b"pipeline: [test]\njobs: {}\n")
+ expected_exception_msg = re.escape(
+ "Job 'test' in pipeline for %s:%s but not in jobs" % (
+ job.build.git_repository.unique_name, job.build.commit_sha1))
+ with ExpectedException(CannotBuild, expected_exception_msg):
+ yield job.composeBuildRequest(None)
+
+ @defer.inlineCallbacks
+ def test_dispatchBuildToWorker_prefers_lxd(self):
+ self.pushConfig("builddmaster", builder_proxy_host=None)
+ job = self.makeJob()
+ builder = MockBuilder()
+ builder.processor = job.build.processor
+ worker = OkWorker()
+ job.setBuilder(builder, worker)
+ chroot_lfa = self.factory.makeLibraryFileAlias(db_only=True)
+ job.build.distro_arch_series.addOrUpdateChroot(
+ chroot_lfa, image_type=BuildBaseImageType.CHROOT)
+ lxd_lfa = self.factory.makeLibraryFileAlias(db_only=True)
+ job.build.distro_arch_series.addOrUpdateChroot(
+ lxd_lfa, image_type=BuildBaseImageType.LXD)
+ yield job.dispatchBuildToWorker(DevNullLogger())
+ self.assertEqual(
+ ("ensurepresent", lxd_lfa.http_url, "", ""), worker.call_log[0])
+ self.assertEqual(1, self.stats_client.incr.call_count)
+ self.assertEqual(
+ self.stats_client.incr.call_args_list[0][0],
+ ("build.count,builder_name={},env=test,"
+ "job_type=CIBUILD".format(builder.name),))
+
+ @defer.inlineCallbacks
+ def test_dispatchBuildToWorker_falls_back_to_chroot(self):
+ self.pushConfig("builddmaster", builder_proxy_host=None)
+ job = self.makeJob()
+ builder = MockBuilder()
+ builder.processor = job.build.processor
+ worker = OkWorker()
+ job.setBuilder(builder, worker)
+ chroot_lfa = self.factory.makeLibraryFileAlias(db_only=True)
+ job.build.distro_arch_series.addOrUpdateChroot(
+ chroot_lfa, image_type=BuildBaseImageType.CHROOT)
+ yield job.dispatchBuildToWorker(DevNullLogger())
+ self.assertEqual(
+ ("ensurepresent", chroot_lfa.http_url, "", ""), worker.call_log[0])
+
+
+class MakeCIBuildMixin:
+ """Provide the common makeBuild method returning a queued build."""
+
+ def makeBuild(self):
+ build = self.factory.makeCIBuild(status=BuildStatus.BUILDING)
+ das = build.distro_arch_series
+ self.useFixture(GitHostingFixture(blob=dedent("""\
+ pipeline:
+ - [test]
+ jobs:
+ test:
+ series: {}
+ architectures: [{}]
+ """.format(das.distroseries.name, das.architecturetag)).encode()))
+ build.queueBuild()
+ return build
+
+ def makeUnmodifiableBuild(self):
+ self.skipTest("Not relevant for CI builds.")
+
+
+class TestGetUploadMethodsForCIBuild(
+ MakeCIBuildMixin, TestGetUploadMethodsMixin, TestCaseWithFactory):
+ """IPackageBuild.getUpload* methods work with CI builds."""
+
+
+class TestVerifySuccessfulBuildForCIBuild(
+ MakeCIBuildMixin, TestVerifySuccessfulBuildMixin, TestCaseWithFactory):
+ """IBuildFarmJobBehaviour.verifySuccessfulBuild works with CI builds."""
+
+
+class TestHandleStatusForCIBuild(
+ MakeCIBuildMixin, TestHandleStatusMixin, TestCaseWithFactory):
+ """IPackageBuild.handleStatus works with CI builds."""
+
+ @defer.inlineCallbacks
+ def test_handleStatus_WAITING_OK_with_jobs(self):
+ # If the worker status includes a "jobs" item, then we additionally
+ # dump that to jobs.json.
+ with dbuser(config.builddmaster.dbuser):
+ yield self.behaviour.handleStatus(
+ self.build.buildqueue_record,
+ {"builder_status": "BuilderStatus.WAITING",
+ "build_status": "BuildStatus.OK",
+ "filemap": {"build:0.log": "test_file_hash"},
+ "jobs": {"build:0": {"log": "test_file_hash"}}})
+ jobs_path = os.path.join(
+ self.upload_root, "incoming",
+ self.behaviour.getUploadDirLeaf(self.build.build_cookie),
+ str(self.build.archive.id), self.build.distribution.name,
+ "jobs.json")
+ with open(jobs_path) as jobs_file:
+ self.assertEqual(
+ {"build:0": {"log": "test_file_hash"}}, json.load(jobs_file))
diff --git a/lib/lp/oci/model/ocirecipebuildbehaviour.py b/lib/lp/oci/model/ocirecipebuildbehaviour.py
index 14cf0ed..6939ca5 100644
--- a/lib/lp/oci/model/ocirecipebuildbehaviour.py
+++ b/lib/lp/oci/model/ocirecipebuildbehaviour.py
@@ -220,11 +220,12 @@ class OCIRecipeBuildBehaviour(BuilderProxyMixin, BuildFarmJobBehaviourBase):
return (filemap[file_name], file_path)
@defer.inlineCallbacks
- def _downloadFiles(self, filemap, upload_path, logger):
+ def _downloadFiles(self, worker_status, upload_path, logger):
"""Download required artifact files."""
+ filemap = worker_status["filemap"]
+
# We don't want to download all of the files that have been created,
# just the ones that are mentioned in the manifest and config.
-
manifest = yield self._fetchIntermediaryFile(
'manifest.json', filemap, upload_path)
digests = yield self._fetchIntermediaryFile(
