launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #28192
[Merge] ~jugmac00/launchpad:create-lpcraft-jobs-on-push into launchpad:master
Jürgen Gmach has proposed merging ~jugmac00/launchpad:create-lpcraft-jobs-on-push into launchpad:master.
Commit message:
WIP: Create lpcraft jobs on push
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~jugmac00/launchpad/+git/launchpad/+merge/416223
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~jugmac00/launchpad:create-lpcraft-jobs-on-push into launchpad:master.
diff --git a/lib/lp/code/interfaces/cibuild.py b/lib/lp/code/interfaces/cibuild.py
index 4537b1c..e1abcf8 100644
--- a/lib/lp/code/interfaces/cibuild.py
+++ b/lib/lp/code/interfaces/cibuild.py
@@ -6,11 +6,16 @@
__all__ = [
"CannotFetchConfiguration",
"CannotParseConfiguration",
+ "CIBuildAlreadyRequested",
+ "CIBuildDisallowedArchitecture",
"ICIBuild",
"ICIBuildSet",
"MissingConfiguration",
]
+import http.client
+
+from lazr.restful.declarations import error_status
from lazr.restful.fields import Reference
from zope.schema import (
Bool,
@@ -53,6 +58,26 @@ class CannotParseConfiguration(Exception):
"""Launchpad cannot parse this CI build's .launchpad.yaml."""
+@error_status(http.client.BAD_REQUEST)
+class CIBuildDisallowedArchitecture(Exception):
+ """A build was requested for a disallowed architecture."""
+
+ def __init__(self, das, pocket):
+ super().__init__(
+ "Builds for %s/%s are not allowed." % (
+ das.distroseries.getSuite(pocket), das.architecturetag)
+ )
+
+
+@error_status(http.client.BAD_REQUEST)
+class CIBuildAlreadyRequested(Exception):
+ """An identical build was requested more than once."""
+
+ def __init__(self):
+ super().__init__(
+ "An identical build for this commit was already requested.")
+
+
class ICIBuildView(IPackageBuildView):
"""`ICIBuild` attributes that require launchpad.View."""
@@ -133,6 +158,37 @@ class ICIBuildSet(ISpecificBuildFarmJobSource):
these Git commit IDs.
"""
+ def requestBuild(git_repository, commit_sha1, distro_arch_series):
+ """Request a CI build.
+
+ This checks that the architecture is allowed and that there isn't
+ already a matching pending build.
+
+ :param git_repository: The `IGitRepository` for the new build.
+ :param commit_sha1: The Git commit ID for the new build.
+ :param distro_arch_series: The `IDistroArchSeries` that the new
+ build should run on.
+ :raises CIBuildDisallowedArchitecture: if builds on
+ `distro_arch_series` are not allowed.
+ :raises CIBuildAlreadyRequested: if a matching build was already
+ requested.
+ :return: `ICIBuild`.
+ """
+
+ def requestBuildsForRefs(git_repository, ref_paths, logger=None):
+ """Request CI builds for a collection of refs.
+
+ This fetches `.launchpad.yaml` from the repository and parses it to
+ work out which series/architectures need builds.
+
+ :param git_repository: The `IGitRepository` for which to request
+ builds.
+ :param ref_paths: A collection of Git reference paths within
+ `git_repository`; builds will be requested for the commits that
+ each of them points to.
+ :param logger: An optional logger.
+ """
+
def deleteByGitRepository(git_repository):
"""Delete all CI builds for the given Git repository.
diff --git a/lib/lp/code/model/cibuild.py b/lib/lp/code/model/cibuild.py
index f9c9031..2407aef 100644
--- a/lib/lp/code/model/cibuild.py
+++ b/lib/lp/code/model/cibuild.py
@@ -9,6 +9,7 @@ __all__ = [
from datetime import timedelta
+from lazr.lifecycle.event import ObjectCreatedEvent
import pytz
from storm.locals import (
Bool,
@@ -21,8 +22,11 @@ from storm.locals import (
)
from storm.store import EmptyResultSet
from zope.component import getUtility
+from zope.event import notify
from zope.interface import implementer
+from lp.app.errors import NotFoundError
+from lp.app.interfaces.launchpad import ILaunchpadCelebrities
from lp.buildmaster.enums import (
BuildFarmJobType,
BuildQueueStatus,
@@ -38,10 +42,14 @@ from lp.code.errors import (
from lp.code.interfaces.cibuild import (
CannotFetchConfiguration,
CannotParseConfiguration,
+ CIBuildAlreadyRequested,
+ CIBuildDisallowedArchitecture,
ICIBuild,
ICIBuildSet,
MissingConfiguration,
)
+from lp.code.interfaces.githosting import IGitHostingClient
+from lp.code.model.gitref import GitRef
from lp.code.model.lpcraft import load_configuration
from lp.registry.interfaces.pocket import PackagePublishingPocket
from lp.registry.interfaces.series import SeriesStatus
@@ -65,6 +73,53 @@ from lp.services.propertycache import cachedproperty
from lp.soyuz.model.distroarchseries import DistroArchSeries
+def determine_DASes_to_build(configuration, logger=None):
+ """Generate distroarchseries to build for this configuration."""
+ architectures_by_series = {}
+ for stage in configuration.pipeline:
+ for job_name in stage:
+ if job_name not in configuration.jobs:
+ if logger is not None:
+ logger.error("No job definition for %r", job_name)
+ continue
+ for job in configuration.jobs[job_name]:
+ for architecture in job["architectures"]:
+ architectures_by_series.setdefault(
+ job["series"], set()).add(architecture)
+ # XXX cjwatson 2022-01-21: We have to hardcode Ubuntu for now, since
+ # the .launchpad.yaml format doesn't currently support other
+ # distributions (although nor does the Launchpad build farm).
+ distribution = getUtility(ILaunchpadCelebrities).ubuntu
+ for series_name, architecture_names in architectures_by_series.items():
+ try:
+ series = distribution[series_name]
+ except NotFoundError:
+ if logger is not None:
+ logger.error("Unknown Ubuntu series name %s" % series_name)
+ continue
+ architectures = {
+ das.architecturetag: das
+ for das in series.buildable_architectures}
+ for architecture_name in architecture_names:
+ try:
+ das = architectures[architecture_name]
+ except KeyError:
+ if logger is not None:
+ logger.error(
+ "%s is not a buildable architecture name in "
+ "Ubuntu %s" % (architecture_name, series_name))
+ continue
+ yield das
+
+
+def get_all_commits_for_paths(git_repository, paths):
+ return [
+ ref.commit_sha1
+ for ref in GitRef.findByReposAndPaths(
+ [(git_repository, ref_path)
+ for ref_path in paths]).values()]
+
+
def parse_configuration(git_repository, blob):
try:
return load_configuration(blob)
@@ -329,6 +384,89 @@ class CIBuildSet(SpecificBuildFarmJobSourceMixin):
store.flush()
return cibuild
+ def findByGitRepository(self, git_repository, commit_sha1s=None):
+ """See `ICIBuildSet`."""
+ clauses = [CIBuild.git_repository == git_repository]
+ if commit_sha1s is not None:
+ clauses.append(CIBuild.commit_sha1.is_in(commit_sha1s))
+ return IStore(CIBuild).find(CIBuild, *clauses)
+
+ def _isBuildableArchitectureAllowed(self, das):
+ """Check whether we may build for a buildable `DistroArchSeries`.
+
+ The caller is assumed to have already checked that a suitable chroot
+ is available (either directly or via
+ `DistroSeries.buildable_architectures`).
+ """
+ return (
+ das.enabled
+ # We only support builds on virtualized builders at the moment.
+ and das.processor.supports_virtualized)
+
+ def _isArchitectureAllowed(self, das, pocket, snap_base=None):
+ return (
+ das.getChroot(pocket=pocket) is not None
+ and self._isBuildableArchitectureAllowed(das))
+
+ def requestBuild(self, git_repository, commit_sha1, distro_arch_series):
+ """See `ICIBuildSet`."""
+ pocket = PackagePublishingPocket.UPDATES
+ if not self._isArchitectureAllowed(distro_arch_series, pocket):
+ raise CIBuildDisallowedArchitecture(distro_arch_series, pocket)
+
+ result = IStore(CIBuild).find(
+ CIBuild,
+ CIBuild.git_repository == git_repository,
+ CIBuild.commit_sha1 == commit_sha1,
+ CIBuild.distro_arch_series == distro_arch_series)
+ if not result.is_empty():
+ raise CIBuildAlreadyRequested
+
+ build = self.new(git_repository, commit_sha1, distro_arch_series)
+ build.queueBuild()
+ notify(ObjectCreatedEvent(build))
+ return build
+
+ def requestBuildsForRefs(self, git_repository, ref_paths, logger=None):
+ """See `ICIBuildSet`."""
+ commit_sha1s = get_all_commits_for_paths(git_repository, ref_paths)
+ # getCommits performs a web request!
+ commits = getUtility(IGitHostingClient).getCommits(
+ git_repository.getInternalPath(), commit_sha1s,
+ # XXX cjwatson 2022-01-19: We should also fetch
+ # debian/.launchpad.yaml (or perhaps make the path a property of
+ # the repository) once lpcraft and launchpad-buildd support
+ # using alternative paths for builds.
+ filter_paths=[".launchpad.yaml"])
+ for commit in commits:
+ try:
+ configuration = parse_configuration(
+ git_repository, commit["blobs"][".launchpad.yaml"])
+ except CannotParseConfiguration as e:
+ if logger is not None:
+ logger.error(e)
+ continue
+ for das in determine_DASes_to_build(configuration):
+ self._tryToRequestBuild(
+ git_repository, commit["sha1"], das, logger)
+
+ def _tryToRequestBuild(self, git_repository, commit_sha1, das, logger):
+ try:
+ if logger is not None:
+ logger.info(
+ "Requesting CI build for %s on %s/%s",
+ commit_sha1, das.distroseries.name, das.architecturetag,
+ )
+ self.requestBuild(git_repository, commit_sha1, das)
+ except CIBuildAlreadyRequested:
+ pass
+ except Exception as e:
+ if logger is not None:
+ logger.error(
+ "Failed to request CI build for %s on %s/%s: %s",
+ commit_sha1, das.distroseries.name, das.architecturetag, e
+ )
+
def getByID(self, build_id):
"""See `ISpecificBuildFarmJobSource`."""
store = IMasterStore(CIBuild)
@@ -357,13 +495,6 @@ class CIBuildSet(SpecificBuildFarmJobSourceMixin):
bfj.id for bfj in build_farm_jobs))
return DecoratedResultSet(rows, pre_iter_hook=self.preloadBuildsData)
- def findByGitRepository(self, git_repository, commit_sha1s=None):
- """See `ICIBuildSet`."""
- clauses = [CIBuild.git_repository == git_repository]
- if commit_sha1s is not None:
- clauses.append(CIBuild.commit_sha1.is_in(commit_sha1s))
- return IStore(CIBuild).find(CIBuild, *clauses)
-
def deleteByGitRepository(self, git_repository):
"""See `ICIBuildSet`."""
self.findByGitRepository(git_repository).remove()
diff --git a/lib/lp/code/model/tests/test_cibuild.py b/lib/lp/code/model/tests/test_cibuild.py
index 133bed2..c8e9721 100644
--- a/lib/lp/code/model/tests/test_cibuild.py
+++ b/lib/lp/code/model/tests/test_cibuild.py
@@ -7,9 +7,13 @@ from datetime import (
datetime,
timedelta,
)
+import hashlib
from textwrap import dedent
+from unittest.mock import Mock
+from fixtures import MockPatchObject
import pytz
+from storm.locals import Store
from testtools.matchers import (
Equals,
MatchesStructure,
@@ -18,9 +22,14 @@ from zope.component import getUtility
from zope.security.proxy import removeSecurityProxy
from lp.app.enums import InformationType
-from lp.buildmaster.enums import BuildStatus
+from lp.app.interfaces.launchpad import ILaunchpadCelebrities
+from lp.buildmaster.enums import (
+ BuildQueueStatus,
+ BuildStatus,
+ )
from lp.buildmaster.interfaces.buildqueue import IBuildQueue
from lp.buildmaster.interfaces.packagebuild import IPackageBuild
+from lp.buildmaster.model.buildqueue import BuildQueue
from lp.code.errors import (
GitRepositoryBlobNotFound,
GitRepositoryScanFault,
@@ -28,12 +37,20 @@ from lp.code.errors import (
from lp.code.interfaces.cibuild import (
CannotFetchConfiguration,
CannotParseConfiguration,
+ CIBuildAlreadyRequested,
+ CIBuildDisallowedArchitecture,
ICIBuild,
ICIBuildSet,
MissingConfiguration,
)
+from lp.code.model.cibuild import (
+ determine_DASes_to_build,
+ get_all_commits_for_paths,
+ )
+from lp.code.model.lpcraft import load_configuration
from lp.code.tests.helpers import GitHostingFixture
from lp.registry.interfaces.series import SeriesStatus
+from lp.services.log.logger import BufferLogger
from lp.services.propertycache import clear_property_cache
from lp.testing import (
person_logged_in,
@@ -336,6 +353,336 @@ class TestCIBuildSet(TestCaseWithFactory):
self.assertContentEqual(
builds[2:], ci_build_set.findByGitRepository(repositories[1]))
+ def test_requestCIBuild(self):
+ # requestBuild creates a new CIBuild.
+ repository = self.factory.makeGitRepository()
+ commit_sha1 = hashlib.sha1(self.factory.getUniqueBytes()).hexdigest()
+ das = self.factory.makeBuildableDistroArchSeries()
+
+ build = getUtility(ICIBuildSet).requestBuild(
+ repository, commit_sha1, das)
+
+ self.assertTrue(ICIBuild.providedBy(build))
+ self.assertThat(build, MatchesStructure.byEquality(
+ git_repository=repository,
+ commit_sha1=commit_sha1,
+ distro_arch_series=das,
+ status=BuildStatus.NEEDSBUILD,
+ ))
+ store = Store.of(build)
+ store.flush()
+ build_queue = store.find(
+ BuildQueue,
+ BuildQueue._build_farm_job_id ==
+ removeSecurityProxy(build).build_farm_job_id).one()
+ self.assertProvides(build_queue, IBuildQueue)
+ self.assertTrue(build_queue.virtualized)
+ self.assertEqual(BuildQueueStatus.WAITING, build_queue.status)
+
+ def test_requestBuild_score(self):
+ # CI builds have an initial queue score of 2600.
+ repository = self.factory.makeGitRepository()
+ commit_sha1 = hashlib.sha1(self.factory.getUniqueBytes()).hexdigest()
+ das = self.factory.makeBuildableDistroArchSeries()
+ build = getUtility(ICIBuildSet).requestBuild(
+ repository, commit_sha1, das)
+ queue_record = build.buildqueue_record
+ queue_record.score()
+ self.assertEqual(2600, queue_record.lastscore)
+
+ def test_requestBuild_rejects_repeats(self):
+ # requestBuild refuses if an identical build was already requested.
+ repository = self.factory.makeGitRepository()
+ commit_sha1 = hashlib.sha1(self.factory.getUniqueBytes()).hexdigest()
+ distro_series = self.factory.makeDistroSeries()
+ arches = [
+ self.factory.makeBuildableDistroArchSeries(
+ distroseries=distro_series)
+ for _ in range(2)]
+ old_build = getUtility(ICIBuildSet).requestBuild(
+ repository, commit_sha1, arches[0])
+ self.assertRaises(
+ CIBuildAlreadyRequested, getUtility(ICIBuildSet).requestBuild,
+ repository, commit_sha1, arches[0])
+ # We can build for a different distroarchseries.
+ getUtility(ICIBuildSet).requestBuild(
+ repository, commit_sha1, arches[1])
+ # Changing the status of the old build does not allow a new build.
+ old_build.updateStatus(BuildStatus.BUILDING)
+ old_build.updateStatus(BuildStatus.FULLYBUILT)
+ self.assertRaises(
+ CIBuildAlreadyRequested, getUtility(ICIBuildSet).requestBuild,
+ repository, commit_sha1, arches[0])
+
+ def test_requestBuild_virtualization(self):
+ # New builds are virtualized.
+ repository = self.factory.makeGitRepository()
+ commit_sha1 = hashlib.sha1(self.factory.getUniqueBytes()).hexdigest()
+ distro_series = self.factory.makeDistroSeries()
+ for proc_nonvirt in True, False:
+ das = self.factory.makeBuildableDistroArchSeries(
+ distroseries=distro_series, supports_virtualized=True,
+ supports_nonvirtualized=proc_nonvirt)
+ build = getUtility(ICIBuildSet).requestBuild(
+ repository, commit_sha1, das)
+ self.assertTrue(build.virtualized)
+
+ def test_requestBuild_nonvirtualized(self):
+ # A non-virtualized processor cannot run a CI build.
+ repository = self.factory.makeGitRepository()
+ commit_sha1 = hashlib.sha1(self.factory.getUniqueBytes()).hexdigest()
+ distro_series = self.factory.makeDistroSeries()
+ das = self.factory.makeBuildableDistroArchSeries(
+ distroseries=distro_series, supports_virtualized=False,
+ supports_nonvirtualized=True)
+ self.assertRaises(
+ CIBuildDisallowedArchitecture,
+ getUtility(ICIBuildSet).requestBuild, repository, commit_sha1, das)
+
+ def test_requestBuildsForRefs_triggers_builds(self):
+ ubuntu = getUtility(ILaunchpadCelebrities).ubuntu
+ series = self.factory.makeDistroSeries(
+ distribution=ubuntu,
+ name="focal",
+ )
+ self.factory.makeBuildableDistroArchSeries(
+ distroseries=series,
+ architecturetag="amd64"
+ )
+ configuration = dedent("""\
+ pipeline:
+ - test
+
+ jobs:
+ test:
+ series: focal
+ architectures: amd64
+ run: echo hello world >output
+ """).encode()
+ repository = self.factory.makeGitRepository()
+ ref_paths = ['refs/heads/master']
+ [ref] = self.factory.makeGitRefs(repository, ref_paths)
+ encoded_commit_json = {
+ "sha1": ref.commit_sha1,
+ "blobs": {".launchpad.yaml": configuration},
+ }
+ hosting_fixture = self.useFixture(
+ GitHostingFixture(commits=[encoded_commit_json])
+ )
+
+ getUtility(ICIBuildSet).requestBuildsForRefs(repository, ref_paths)
+
+ self.assertEqual(
+ [((repository.getInternalPath(), [ref.commit_sha1]),
+ {"filter_paths": [".launchpad.yaml"]})],
+ hosting_fixture.getCommits.calls
+ )
+
+ build = getUtility(ICIBuildSet).findByGitRepository(repository).one()
+
+ # check that a build was created
+ self.assertEqual(ref.commit_sha1, build.commit_sha1)
+ self.assertEqual("focal", build.distro_arch_series.distroseries.name)
+ self.assertEqual("amd64", build.distro_arch_series.architecturetag)
+
+ def test_requestBuildsForRefs_no_commits_at_all(self):
+ repository = self.factory.makeGitRepository()
+ ref_paths = ['refs/heads/master']
+ hosting_fixture = self.useFixture(GitHostingFixture(commits=[]))
+
+ getUtility(ICIBuildSet).requestBuildsForRefs(repository, ref_paths)
+
+ self.assertEqual(
+ [((repository.getInternalPath(), []),
+ {"filter_paths": [".launchpad.yaml"]})],
+ hosting_fixture.getCommits.calls
+ )
+
+ build = getUtility(ICIBuildSet).findByGitRepository(repository).one()
+
+ self.assertIsNone(build)
+
+ def test_requestBuildsForRefs_no_matching_commits(self):
+ repository = self.factory.makeGitRepository()
+ ref_paths = ['refs/heads/master']
+ [ref] = self.factory.makeGitRefs(repository, ref_paths)
+ hosting_fixture = self.useFixture(
+ GitHostingFixture(commits=[])
+ )
+
+ getUtility(ICIBuildSet).requestBuildsForRefs(repository, ref_paths)
+
+ self.assertEqual(
+ [((repository.getInternalPath(), [ref.commit_sha1]),
+ {"filter_paths": [".launchpad.yaml"]})],
+ hosting_fixture.getCommits.calls
+ )
+
+ build = getUtility(ICIBuildSet).findByGitRepository(repository).one()
+
+ self.assertIsNone(build)
+
+ def test_requestBuildsForRefs_configuration_parse_error(self):
+ ubuntu = getUtility(ILaunchpadCelebrities).ubuntu
+ series = self.factory.makeDistroSeries(
+ distribution=ubuntu,
+ name="focal",
+ )
+ self.factory.makeBuildableDistroArchSeries(
+ distroseries=series,
+ architecturetag="amd64"
+ )
+ configuration = dedent("""\
+ no - valid - configuration - file
+ """).encode()
+ repository = self.factory.makeGitRepository()
+ ref_paths = ['refs/heads/master']
+ [ref] = self.factory.makeGitRefs(repository, ref_paths)
+ encoded_commit_json = {
+ "sha1": ref.commit_sha1,
+ "blobs": {".launchpad.yaml": configuration},
+ }
+ hosting_fixture = self.useFixture(
+ GitHostingFixture(commits=[encoded_commit_json])
+ )
+ logger = BufferLogger()
+
+ getUtility(ICIBuildSet).requestBuildsForRefs(
+ repository, ref_paths, logger)
+
+ self.assertEqual(
+ [((repository.getInternalPath(), [ref.commit_sha1]),
+ {"filter_paths": [".launchpad.yaml"]})],
+ hosting_fixture.getCommits.calls
+ )
+
+ build = getUtility(ICIBuildSet).findByGitRepository(repository).one()
+
+ self.assertIsNone(build)
+
+ self.assertEqual(
+ "ERROR Cannot parse .launchpad.yaml from %s: "
+ "Configuration file does not declare 'pipeline'\n" % (
+ repository.unique_name,),
+ logger.getLogBuffer()
+ )
+
+ def test_requestBuildsForRefs_build_already_scheduled(self):
+ ubuntu = getUtility(ILaunchpadCelebrities).ubuntu
+ series = self.factory.makeDistroSeries(
+ distribution=ubuntu,
+ name="focal",
+ )
+ self.factory.makeBuildableDistroArchSeries(
+ distroseries=series,
+ architecturetag="amd64"
+ )
+ configuration = dedent("""\
+ pipeline:
+ - test
+
+ jobs:
+ test:
+ series: focal
+ architectures: amd64
+ run: echo hello world >output
+ """).encode()
+ repository = self.factory.makeGitRepository()
+ ref_paths = ['refs/heads/master']
+ [ref] = self.factory.makeGitRefs(repository, ref_paths)
+ encoded_commit_json = {
+ "sha1": ref.commit_sha1,
+ "blobs": {".launchpad.yaml": configuration},
+ }
+ hosting_fixture = self.useFixture(
+ GitHostingFixture(commits=[encoded_commit_json])
+ )
+ build_set = removeSecurityProxy(getUtility(ICIBuildSet))
+ mock = Mock(side_effect=CIBuildAlreadyRequested)
+ self.useFixture(MockPatchObject(build_set, "requestBuild", mock))
+ logger = BufferLogger()
+
+ build_set.requestBuildsForRefs(repository, ref_paths, logger)
+
+ self.assertEqual(
+ [((repository.getInternalPath(), [ref.commit_sha1]),
+ {"filter_paths": [".launchpad.yaml"]})],
+ hosting_fixture.getCommits.calls
+ )
+
+ build = getUtility(ICIBuildSet).findByGitRepository(repository).one()
+
+ self.assertIsNone(build)
+
+ # XXX jugmac00 2022-03-04
+ # for unknown reasons, the logger output starts with a backslash:
+ #
+ # actual = '''\
+ # INFO Requesting CI build for 972c6d2dc6dd5efdad1377c0d224e03eb8f276f7 on focal/amd64 # noqa: E501
+ # '''
+ self.assertIn(
+ "INFO Requesting CI build for %s on focal/amd64" % ref.commit_sha1,
+ logger.getLogBuffer()
+ )
+
+ def test_requestBuildsForRefs_unexpected_exception(self):
+ ubuntu = getUtility(ILaunchpadCelebrities).ubuntu
+ series = self.factory.makeDistroSeries(
+ distribution=ubuntu,
+ name="focal",
+ )
+ self.factory.makeBuildableDistroArchSeries(
+ distroseries=series,
+ architecturetag="amd64"
+ )
+ configuration = dedent("""\
+ pipeline:
+ - test
+
+ jobs:
+ test:
+ series: focal
+ architectures: amd64
+ run: echo hello world >output
+ """).encode()
+ repository = self.factory.makeGitRepository()
+ ref_paths = ['refs/heads/master']
+ [ref] = self.factory.makeGitRefs(repository, ref_paths)
+ encoded_commit_json = {
+ "sha1": ref.commit_sha1,
+ "blobs": {".launchpad.yaml": configuration},
+ }
+ hosting_fixture = self.useFixture(
+ GitHostingFixture(commits=[encoded_commit_json])
+ )
+ build_set = removeSecurityProxy(getUtility(ICIBuildSet))
+ mock = Mock(side_effect=Exception("some unexpected error"))
+ self.useFixture(MockPatchObject(build_set, "requestBuild", mock))
+ logger = BufferLogger()
+
+ build_set.requestBuildsForRefs(repository, ref_paths, logger)
+
+ self.assertEqual(
+ [((repository.getInternalPath(), [ref.commit_sha1]),
+ {"filter_paths": [".launchpad.yaml"]})],
+ hosting_fixture.getCommits.calls
+ )
+
+ build = getUtility(ICIBuildSet).findByGitRepository(repository).one()
+
+ self.assertIsNone(build)
+
+ # last line is an empty string
+ log_line1, log_line2, _ = logger.getLogBuffer().split("\n")
+ self.assertEqual(
+ "INFO Requesting CI build for %s on focal/amd64" % ref.commit_sha1,
+ log_line1)
+ self.assertEqual(
+ "ERROR Failed to request CI build for %s on focal/amd64: "
+ "some unexpected error" % (ref.commit_sha1,),
+ log_line2
+ )
+
def test_deleteByGitRepository(self):
repositories = [self.factory.makeGitRepository() for _ in range(2)]
builds = []
@@ -350,3 +697,157 @@ class TestCIBuildSet(TestCaseWithFactory):
[], ci_build_set.findByGitRepository(repositories[0]))
self.assertContentEqual(
builds[2:], ci_build_set.findByGitRepository(repositories[1]))
+
+
+class TestCIBuildHelpers(TestCaseWithFactory):
+
+ layer = LaunchpadZopelessLayer
+
+ def test_determine_DASes_to_build(self):
+ ubuntu = getUtility(ILaunchpadCelebrities).ubuntu
+ distro_serieses = [
+ self.factory.makeDistroSeries(ubuntu) for _ in range(2)]
+ dases = []
+ for distro_series in distro_serieses:
+ for _ in range(2):
+ dases.append(self.factory.makeBuildableDistroArchSeries(
+ distroseries=distro_series))
+ configuration = load_configuration(dedent("""\
+ pipeline:
+ - [build]
+ - [test]
+ jobs:
+ build:
+ series: {distro_serieses[1].name}
+ architectures:
+ - {dases[2].architecturetag}
+ - {dases[3].architecturetag}
+ test:
+ series: {distro_serieses[1].name}
+ architectures:
+ - {dases[2].architecturetag}
+ """.format(distro_serieses=distro_serieses, dases=dases)))
+ logger = BufferLogger()
+
+ dases_to_build = list(
+ determine_DASes_to_build(configuration, logger=logger))
+
+ self.assertContentEqual(dases[2:], dases_to_build)
+ self.assertEqual("", logger.getLogBuffer())
+
+
+ def test_determine_DASes_to_build_logs_missing_job_definition(self):
+ ubuntu = getUtility(ILaunchpadCelebrities).ubuntu
+ distro_series = self.factory.makeDistroSeries(ubuntu)
+ das = self.factory.makeBuildableDistroArchSeries(
+ distroseries=distro_series)
+ configuration = load_configuration(dedent("""\
+ pipeline:
+ - [test]
+ jobs:
+ build:
+ series: {distro_series.name}
+ architectures:
+ - {das.architecturetag}
+ """.format(distro_series=distro_series, das=das)))
+ logger = BufferLogger()
+
+ dases_to_build = list(
+ determine_DASes_to_build(configuration, logger=logger))
+
+ self.assertEqual(0, len(dases_to_build))
+ self.assertEqual(
+ "ERROR No job definition for 'test'\n", logger.getLogBuffer()
+ )
+
+
+ def test_determine_DASes_to_build_logs_missing_series(self):
+ ubuntu = getUtility(ILaunchpadCelebrities).ubuntu
+ distro_series = self.factory.makeDistroSeries(ubuntu)
+ das = self.factory.makeBuildableDistroArchSeries(
+ distroseries=distro_series)
+ configuration = load_configuration(dedent("""\
+ pipeline:
+ - [build]
+ jobs:
+ build:
+ series: unknown-series
+ architectures:
+ - {das.architecturetag}
+ """.format(das=das)))
+ logger = BufferLogger()
+
+ dases_to_build = list(
+ determine_DASes_to_build(configuration, logger=logger))
+
+ self.assertEqual(0, len(dases_to_build))
+ self.assertEqual(
+ "ERROR Unknown Ubuntu series name unknown-series\n",
+ logger.getLogBuffer()
+ )
+
+
+ def test_determine_DASes_to_build_logs_non_buildable_architecture(self):
+ ubuntu = getUtility(ILaunchpadCelebrities).ubuntu
+ distro_series = self.factory.makeDistroSeries(ubuntu)
+ configuration = load_configuration(dedent("""\
+ pipeline:
+ - [build]
+ jobs:
+ build:
+ series: {distro_series.name}
+ architectures:
+ - non-buildable-architecture
+ """.format(distro_series=distro_series)))
+ logger = BufferLogger()
+
+ dases_to_build = list(
+ determine_DASes_to_build(configuration, logger=logger))
+
+ self.assertEqual(0, len(dases_to_build))
+ # XXX jugmac00 2022-03-08
+ # for unknown reasons, the logger output starts with a backslash:
+ # actual = '''\
+ # ERROR non-buildable-architecture is not a buildable architecture name in Ubuntu distroseries-100005 # noqa: E501
+ # '''
+ self.assertIn(
+ "ERROR non-buildable-architecture is not a buildable architecture "
+ "name in Ubuntu %s" % distro_series.name,
+ logger.getLogBuffer()
+ )
+
+
+
+class TestGetAllCommitsForPaths(TestCaseWithFactory):
+
+ layer = LaunchpadZopelessLayer
+
+ def test_no_refs(self):
+ repository = self.factory.makeGitRepository()
+ ref_paths = ['refs/heads/master']
+
+ rv = get_all_commits_for_paths(repository, ref_paths)
+
+ self.assertEqual([], rv)
+
+ def test_one_ref_one_path(self):
+ repository = self.factory.makeGitRepository()
+ ref_paths = ['refs/heads/master']
+ [ref] = self.factory.makeGitRefs(repository, ref_paths)
+
+ rv = get_all_commits_for_paths(repository, ref_paths)
+
+ self.assertEqual(1, len(rv))
+ self.assertEqual(ref.commit_sha1, rv[0])
+
+ def test_multiple_refs_and_paths(self):
+ # XXX jugmac00 2022-03-04
+ # this test possibly should have multiple commits per path
+ repository = self.factory.makeGitRepository()
+ ref_paths = ['refs/heads/master', "refs/heads/dev"]
+ refs = self.factory.makeGitRefs(repository, ref_paths)
+
+ rv = get_all_commits_for_paths(repository, ref_paths)
+
+ self.assertEqual(2, len(rv))
+ self.assertEqual({ref.commit_sha1 for ref in refs}, set(rv))
diff --git a/lib/lp/code/model/tests/test_gitrepository.py b/lib/lp/code/model/tests/test_gitrepository.py
index 3272b3b..bc17444 100644
--- a/lib/lp/code/model/tests/test_gitrepository.py
+++ b/lib/lp/code/model/tests/test_gitrepository.py
@@ -11,6 +11,7 @@ import email
from functools import partial
import hashlib
import json
+from textwrap import dedent
from breezy import urlutils
from fixtures import MockPatch
@@ -81,6 +82,10 @@ from lp.code.event.git import GitRefsUpdatedEvent
from lp.code.interfaces.branchmergeproposal import (
BRANCH_MERGE_PROPOSAL_FINAL_STATES as FINAL_STATES,
)
+from lp.code.interfaces.cibuild import (
+ ICIBuild,
+ ICIBuildSet,
+ )
from lp.code.interfaces.codeimport import ICodeImportSet
from lp.code.interfaces.defaultgit import ICanHasDefaultGitRepository
from lp.code.interfaces.gitjob import (
@@ -161,6 +166,7 @@ from lp.services.identity.interfaces.account import AccountStatus
from lp.services.job.interfaces.job import JobStatus
from lp.services.job.model.job import Job
from lp.services.job.runner import JobRunner
+from lp.services.log.logger import BufferLogger
from lp.services.macaroons.interfaces import IMacaroonIssuer
from lp.services.macaroons.testing import (
find_caveats_by_name,
@@ -1461,6 +1467,7 @@ class TestGitRepositoryModifications(TestCaseWithFactory):
repository, "date_last_modified", UTC_NOW)
def test_create_ref_sets_date_last_modified(self):
+ self.useFixture(GitHostingFixture())
repository = self.factory.makeGitRepository(
date_created=datetime(2015, 6, 1, tzinfo=pytz.UTC))
[ref] = self.factory.makeGitRefs(repository=repository)
@@ -1869,6 +1876,7 @@ class TestGitRepositoryRefs(TestCaseWithFactory):
repository.refs, repository, ["refs/heads/master"])
def test_update(self):
+ self.useFixture(GitHostingFixture())
repository = self.factory.makeGitRepository()
paths = ("refs/heads/master", "refs/tags/1.0")
self.factory.makeGitRefs(repository=repository, paths=paths)
@@ -1900,6 +1908,7 @@ class TestGitRepositoryRefs(TestCaseWithFactory):
return [UpdatePreviewDiffJob(job) for job in jobs]
def test_update_schedules_diff_update(self):
+ self.useFixture(GitHostingFixture())
repository = self.factory.makeGitRepository()
[ref] = self.factory.makeGitRefs(repository=repository)
self.assertRefsMatch(repository.refs, repository, [ref.path])
@@ -2210,6 +2219,7 @@ class TestGitRepositoryRefs(TestCaseWithFactory):
def test_synchroniseRefs(self):
# synchroniseRefs copes with synchronising a repository where some
# refs have been created, some deleted, and some changed.
+ self.useFixture(GitHostingFixture())
repository = self.factory.makeGitRepository()
paths = ("refs/heads/master", "refs/heads/foo", "refs/heads/bar")
self.factory.makeGitRefs(repository=repository, paths=paths)
@@ -2958,6 +2968,7 @@ class TestGitRepositoryMarkRecipesStale(TestCaseWithFactory):
def test_base_repository_recipe(self):
# On ref changes, recipes where this ref is the base become stale.
+ self.useFixture(GitHostingFixture())
[ref] = self.factory.makeGitRefs()
recipe = self.factory.makeSourcePackageRecipe(branches=[ref])
removeSecurityProxy(recipe).is_stale = False
@@ -2968,6 +2979,7 @@ class TestGitRepositoryMarkRecipesStale(TestCaseWithFactory):
def test_base_repository_different_ref_recipe(self):
# On ref changes, recipes where a different ref in the same
# repository is the base are left alone.
+ self.useFixture(GitHostingFixture())
ref1, ref2 = self.factory.makeGitRefs(
paths=["refs/heads/a", "refs/heads/b"])
recipe = self.factory.makeSourcePackageRecipe(branches=[ref1])
@@ -2979,6 +2991,7 @@ class TestGitRepositoryMarkRecipesStale(TestCaseWithFactory):
def test_base_repository_default_branch_recipe(self):
# On ref changes to the default branch, recipes where this
# repository is the base with no explicit revspec become stale.
+ self.useFixture(GitHostingFixture())
repository = self.factory.makeGitRepository()
ref1, ref2 = self.factory.makeGitRefs(
repository=repository, paths=["refs/heads/a", "refs/heads/b"])
@@ -2994,6 +3007,7 @@ class TestGitRepositoryMarkRecipesStale(TestCaseWithFactory):
def test_instruction_repository_recipe(self):
# On ref changes, recipes including this ref become stale.
+ self.useFixture(GitHostingFixture())
[base_ref] = self.factory.makeGitRefs()
[ref] = self.factory.makeGitRefs()
recipe = self.factory.makeSourcePackageRecipe(branches=[base_ref, ref])
@@ -3005,6 +3019,7 @@ class TestGitRepositoryMarkRecipesStale(TestCaseWithFactory):
def test_instruction_repository_different_ref_recipe(self):
# On ref changes, recipes including a different ref in the same
# repository are left alone.
+ self.useFixture(GitHostingFixture())
[base_ref] = self.factory.makeGitRefs()
ref1, ref2 = self.factory.makeGitRefs(
paths=["refs/heads/a", "refs/heads/b"])
@@ -3018,6 +3033,7 @@ class TestGitRepositoryMarkRecipesStale(TestCaseWithFactory):
def test_instruction_repository_default_branch_recipe(self):
# On ref changes to the default branch, recipes including this
# repository with no explicit revspec become stale.
+ self.useFixture(GitHostingFixture())
[base_ref] = self.factory.makeGitRefs()
repository = self.factory.makeGitRepository()
ref1, ref2 = self.factory.makeGitRefs(
@@ -3035,6 +3051,7 @@ class TestGitRepositoryMarkRecipesStale(TestCaseWithFactory):
def test_unrelated_repository_recipe(self):
# On ref changes, unrelated recipes are left alone.
+ self.useFixture(GitHostingFixture())
[ref] = self.factory.makeGitRefs()
recipe = self.factory.makeSourcePackageRecipe(
branches=self.factory.makeGitRefs())
@@ -3050,6 +3067,7 @@ class TestGitRepositoryMarkSnapsStale(TestCaseWithFactory):
def test_same_repository(self):
# On ref changes, snap packages using this ref become stale.
+ self.useFixture(GitHostingFixture())
[ref] = self.factory.makeGitRefs()
snap = self.factory.makeSnap(git_ref=ref)
removeSecurityProxy(snap).is_stale = False
@@ -3060,6 +3078,7 @@ class TestGitRepositoryMarkSnapsStale(TestCaseWithFactory):
def test_same_repository_different_ref(self):
# On ref changes, snap packages using a different ref in the same
# repository are left alone.
+ self.useFixture(GitHostingFixture())
ref1, ref2 = self.factory.makeGitRefs(
paths=["refs/heads/a", "refs/heads/b"])
snap = self.factory.makeSnap(git_ref=ref1)
@@ -3070,6 +3089,7 @@ class TestGitRepositoryMarkSnapsStale(TestCaseWithFactory):
def test_different_repository(self):
# On ref changes, unrelated snap packages are left alone.
+ self.useFixture(GitHostingFixture())
[ref] = self.factory.makeGitRefs()
snap = self.factory.makeSnap(git_ref=self.factory.makeGitRefs()[0])
removeSecurityProxy(snap).is_stale = False
@@ -3079,6 +3099,7 @@ class TestGitRepositoryMarkSnapsStale(TestCaseWithFactory):
def test_private_snap(self):
# A private snap should be able to be marked stale
+ self.useFixture(GitHostingFixture())
self.useFixture(FeatureFixture(SNAP_TESTING_FLAGS))
[ref] = self.factory.makeGitRefs()
snap = self.factory.makeSnap(git_ref=ref, private=True)
@@ -3101,6 +3122,7 @@ class TestGitRepositoryMarkCharmRecipesStale(TestCaseWithFactory):
def test_same_repository(self):
# On ref changes, charm recipes using this ref become stale.
+ self.useFixture(GitHostingFixture())
[ref] = self.factory.makeGitRefs()
recipe = self.factory.makeCharmRecipe(git_ref=ref)
removeSecurityProxy(recipe).is_stale = False
@@ -3111,6 +3133,7 @@ class TestGitRepositoryMarkCharmRecipesStale(TestCaseWithFactory):
def test_same_repository_different_ref(self):
# On ref changes, charm recipes using a different ref in the same
# repository are left alone.
+ self.useFixture(GitHostingFixture())
ref1, ref2 = self.factory.makeGitRefs(
paths=["refs/heads/a", "refs/heads/b"])
recipe = self.factory.makeCharmRecipe(git_ref=ref1)
@@ -3121,6 +3144,7 @@ class TestGitRepositoryMarkCharmRecipesStale(TestCaseWithFactory):
def test_different_repository(self):
# On ref changes, unrelated charm recipes are left alone.
+ self.useFixture(GitHostingFixture())
[ref] = self.factory.makeGitRefs()
recipe = self.factory.makeCharmRecipe(
git_ref=self.factory.makeGitRefs()[0])
@@ -3321,6 +3345,81 @@ class TestGitRepositoryDetectMerges(TestCaseWithFactory):
for event in events[:2]})
+class TestGitRepositoryRequestCIBuilds(TestCaseWithFactory):
+
+ layer = ZopelessDatabaseLayer
+
+ def test_findByGitRepository_with_configuration(self):
+ # If a changed ref has CI configuration, we request CI builds.
+ logger = BufferLogger()
+ [ref] = self.factory.makeGitRefs()
+ ubuntu = getUtility(ILaunchpadCelebrities).ubuntu
+ distroseries = self.factory.makeDistroSeries(distribution=ubuntu)
+ dases = [
+ self.factory.makeBuildableDistroArchSeries(
+ distroseries=distroseries)
+ for _ in range(2)]
+ configuration = dedent("""\
+ pipeline: [test]
+ jobs:
+ test:
+ series: {series}
+ architectures: [{architectures}]
+ """.format(
+ series=distroseries.name,
+ architectures=", ".join(
+ das.architecturetag for das in dases))).encode()
+ new_commit = hashlib.sha1(self.factory.getUniqueBytes()).hexdigest()
+ self.useFixture(GitHostingFixture(commits=[
+ {
+ "sha1": new_commit,
+ "blobs": {".launchpad.yaml": configuration},
+ },
+ ]))
+ with dbuser("branchscanner"):
+ ref.repository.createOrUpdateRefs(
+ {ref.path: {"sha1": new_commit, "type": GitObjectType.COMMIT}},
+ logger=logger)
+
+ results = getUtility(ICIBuildSet).findByGitRepository(ref.repository)
+ for result in results:
+ self.assertTrue(ICIBuild.providedBy(result))
+
+ self.assertThat(
+ results,
+ MatchesSetwise(*(
+ MatchesStructure.byEquality(
+ git_repository=ref.repository,
+ commit_sha1=new_commit,
+ distro_arch_series=das)
+ for das in dases)))
+ self.assertContentEqual(
+ [
+ "INFO Requesting CI build for {commit} on "
+ "{series}/{arch}".format(
+ commit=new_commit, series=distroseries.name,
+ arch=das.architecturetag)
+ for das in dases],
+ logger.getLogBuffer().splitlines())
+
+ def test_findByGitRepository_without_configuration(self):
+ # If a changed ref has no CI configuration, we do not request CI
+ # builds.
+ logger = BufferLogger()
+ [ref] = self.factory.makeGitRefs()
+ new_commit = hashlib.sha1(self.factory.getUniqueBytes()).hexdigest()
+ self.useFixture(GitHostingFixture(commits=[]))
+ with dbuser("branchscanner"):
+ ref.repository.createOrUpdateRefs(
+ {ref.path: {"sha1": new_commit, "type": GitObjectType.COMMIT}},
+ logger=logger)
+ self.assertTrue(
+ getUtility(
+ ICIBuildSet).findByGitRepository(ref.repository).is_empty()
+ )
+ self.assertEqual("", logger.getLogBuffer())
+
+
class TestGitRepositoryGetBlob(TestCaseWithFactory):
"""Tests for retrieving files from a Git repository."""
diff --git a/lib/lp/code/subscribers/git.py b/lib/lp/code/subscribers/git.py
index 30b432e..831ac49 100644
--- a/lib/lp/code/subscribers/git.py
+++ b/lib/lp/code/subscribers/git.py
@@ -1,8 +1,13 @@
-# Copyright 2015-2016 Canonical Ltd. This software is licensed under the
+# Copyright 2015-2022 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Event subscribers for Git repositories."""
+from zope.component import getUtility
+
+from lp.code.interfaces.cibuild import ICIBuildSet
+
+
def refs_updated(repository, event):
"""Some references in a Git repository have been updated."""
repository.updateMergeCommitIDs(event.paths)
@@ -11,3 +16,5 @@ def refs_updated(repository, event):
repository.markSnapsStale(event.paths)
repository.markCharmRecipesStale(event.paths)
repository.detectMerges(event.paths, logger=event.logger)
+ getUtility(ICIBuildSet).requestBuildsForRefs(
+ repository, event.paths, logger=event.logger)
diff --git a/lib/lp/testing/factory.py b/lib/lp/testing/factory.py
index 041cb89..83ad621 100644
--- a/lib/lp/testing/factory.py
+++ b/lib/lp/testing/factory.py
@@ -97,11 +97,15 @@ from lp.bugs.interfaces.cve import (
)
from lp.bugs.model.bug import FileBugData
from lp.buildmaster.enums import (
+ BuildBaseImageType,
BuilderResetProtocol,
BuildStatus,
)
from lp.buildmaster.interfaces.builder import IBuilderSet
-from lp.buildmaster.interfaces.processor import IProcessorSet
+from lp.buildmaster.interfaces.processor import (
+ IProcessorSet,
+ ProcessorNotFound,
+ )
from lp.charms.interfaces.charmbase import ICharmBaseSet
from lp.charms.interfaces.charmrecipe import ICharmRecipeSet
from lp.charms.interfaces.charmrecipebuild import ICharmRecipeBuildSet
@@ -2909,6 +2913,34 @@ class BareLaunchpadObjectFactory(ObjectFactory):
return distroseries.newArch(
architecturetag, processor, official, owner, enabled)
+ def makeBuildableDistroArchSeries(self, architecturetag=None,
+ processor=None,
+ supports_virtualized=True,
+ supports_nonvirtualized=True, **kwargs):
+ if architecturetag is None:
+ architecturetag = self.getUniqueUnicode("arch")
+ if processor is None:
+ try:
+ processor = getUtility(IProcessorSet).getByName(
+ architecturetag)
+ except ProcessorNotFound:
+ processor = self.makeProcessor(
+ name=architecturetag,
+ supports_virtualized=supports_virtualized,
+ supports_nonvirtualized=supports_nonvirtualized)
+ das = self.makeDistroArchSeries(
+ architecturetag=architecturetag, processor=processor, **kwargs)
+ # Add both a chroot and a LXD image to test that
+ # getAllowedArchitectures doesn't get confused by multiple
+ # PocketChroot rows for a single DistroArchSeries.
+ fake_chroot = self.makeLibraryFileAlias(
+ filename="fake_chroot.tar.gz", db_only=True)
+ das.addOrUpdateChroot(fake_chroot)
+ fake_lxd = self.makeLibraryFileAlias(
+ filename="fake_lxd.tar.gz", db_only=True)
+ das.addOrUpdateChroot(fake_lxd, image_type=BuildBaseImageType.LXD)
+ return das
+
def makeComponent(self, name=None):
"""Make a new `IComponent`."""
if name is None:
Follow ups
