launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #31342
[Merge] ~ruinedyourlife/launchpad-buildd:feat-source-build-base into launchpad-buildd:master
Quentin Debhi has proposed merging ~ruinedyourlife/launchpad-buildd:feat-source-build-base into launchpad-buildd:master.
Commit message:
Add the capability to build sources
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~ruinedyourlife/launchpad-buildd/+git/launchpad-buildd/+merge/471812
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~ruinedyourlife/launchpad-buildd:feat-source-build-base into launchpad-buildd:master.
diff --git a/debian/changelog b/debian/changelog
index 6897374..23fb469 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+launchpad-buildd (242) UNRELEASED; urgency=medium
+
+ [ Quentin Debhi ]
+ * Add sourcecraft build type.
+
+ -- Quentin Debhi <quentin.debhi@xxxxxxxxxxxxx> Thu, 22 Aug 2024 14:09:58 +0200
+
launchpad-buildd (241) focal; urgency=medium
[ Quentin Debhi ]
diff --git a/lpbuildd/buildd.tac b/lpbuildd/buildd.tac
index 41da099..7e968eb 100644
--- a/lpbuildd/buildd.tac
+++ b/lpbuildd/buildd.tac
@@ -23,6 +23,7 @@ from lpbuildd.livefs import LiveFilesystemBuildManager
from lpbuildd.log import RotatableFileLogObserver
from lpbuildd.oci import OCIBuildManager
from lpbuildd.rock import RockBuildManager
+from lpbuildd.source import SourceBuildManager
from lpbuildd.snap import SnapBuildManager
from lpbuildd.sourcepackagerecipe import SourcePackageRecipeBuildManager
from lpbuildd.translationtemplates import TranslationTemplatesBuildManager
@@ -47,6 +48,7 @@ builder.registerManager(OCIBuildManager, "oci")
builder.registerManager(CharmBuildManager, "charm")
builder.registerManager(CIBuildManager, "ci")
builder.registerManager(RockBuildManager, "rock")
+builder.registerManager(SourceBuildManager, "source")
application = service.Application("Builder")
application.addComponent(
diff --git a/lpbuildd/source.py b/lpbuildd/source.py
new file mode 100644
index 0000000..b0aac67
--- /dev/null
+++ b/lpbuildd/source.py
@@ -0,0 +1,96 @@
+import os
+
+from lpbuildd.debian import DebianBuildManager, DebianBuildState
+from lpbuildd.proxy import BuildManagerProxyMixin
+
+RETCODE_SUCCESS = 0
+RETCODE_FAILURE_INSTALL = 200
+RETCODE_FAILURE_BUILD = 201
+
+
+class SourceBuildState(DebianBuildState):
+ BUILD_SOURCE = "BUILD_SOURCE"
+
+
+class SourceBuildManager(BuildManagerProxyMixin, DebianBuildManager):
+ """Build a source."""
+
+ backend_name = "lxd"
+ initial_build_state = SourceBuildState.BUILD_SOURCE
+
+ @property
+ def needs_sanitized_logs(self):
+ return True
+
+ def initiate(self, files, chroot, extra_args):
+ """Initiate a build with a given set of files and chroot."""
+ self.name = extra_args["name"]
+ self.branch = extra_args.get("branch")
+ self.git_repository = extra_args.get("git_repository")
+ self.git_path = extra_args.get("git_path")
+ self.build_path = extra_args.get("build_path")
+ self.channels = extra_args.get("channels", {})
+ self.proxy_url = extra_args.get("proxy_url")
+ self.revocation_endpoint = extra_args.get("revocation_endpoint")
+ self.proxy_service = None
+
+ super().initiate(files, chroot, extra_args)
+
+ def doRunBuild(self):
+ """Run the process to build the source."""
+ args = []
+ args.extend(self.startProxy())
+ if self.revocation_endpoint:
+ args.extend(["--revocation-endpoint", self.revocation_endpoint])
+ for snap, channel in sorted(self.channels.items()):
+ args.extend(["--channel", f"{snap}={channel}"])
+ if self.branch is not None:
+ args.extend(["--branch", self.branch])
+ if self.git_repository is not None:
+ args.extend(["--git-repository", self.git_repository])
+ if self.git_path is not None:
+ args.extend(["--git-path", self.git_path])
+ if self.build_path is not None:
+ args.extend(["--build-path", self.build_path])
+ args.append(self.name)
+ self.runTargetSubProcess("build-source", *args)
+
+ def iterate_BUILD_SOURCE(self, retcode):
+ """Finished building the source."""
+ self.stopProxy()
+ self.revokeProxyToken()
+ if retcode == RETCODE_SUCCESS:
+ print("[source] Returning build status: OK")
+ return self.deferGatherResults()
+ elif (
+ retcode >= RETCODE_FAILURE_INSTALL
+ and retcode <= RETCODE_FAILURE_BUILD
+ ):
+ if not self.alreadyfailed:
+ self._builder.buildFail()
+ print("[source] Returning build status: Builder failed.")
+ self.alreadyfailed = True
+ else:
+ if not self.alreadyfailed:
+ self._builder.buildFail()
+ print("[source] Returning build status: Build failed.")
+ self.alreadyfailed = True
+ self.doReapProcesses(self._state)
+
+ def iterateReap_BUILD_SOURCE(self, retcode):
+ """Finished reaping after building the source."""
+ self._state = DebianBuildState.UMOUNT
+ self.doUnmounting()
+
+ def gatherResults(self):
+ """Gather the results of the build and add them to the file cache."""
+ output_path = os.path.join("/home/buildd", self.name)
+ if self.build_path is not None:
+ output_path = os.path.join(output_path, self.build_path)
+ if self.backend.path_exists(output_path):
+ for entry in sorted(self.backend.listdir(output_path)):
+ path = os.path.join(output_path, entry)
+ if self.backend.islink(path):
+ continue
+ #
+ self.addWaitingFileFromBackend(path)
diff --git a/lpbuildd/target/build_source.py b/lpbuildd/target/build_source.py
new file mode 100644
index 0000000..370371c
--- /dev/null
+++ b/lpbuildd/target/build_source.py
@@ -0,0 +1,120 @@
+import logging
+import os
+
+from lpbuildd.target.backend import check_path_escape
+from lpbuildd.target.build_snap import SnapChannelsAction
+from lpbuildd.target.operation import Operation
+from lpbuildd.target.proxy import BuilderProxyOperationMixin
+from lpbuildd.target.snapstore import SnapStoreOperationMixin
+from lpbuildd.target.vcs import VCSOperationMixin
+
+RETCODE_FAILURE_INSTALL = 200
+RETCODE_FAILURE_BUILD = 201
+
+
+logger = logging.getLogger(__name__)
+
+
+class BuildSource(
+ BuilderProxyOperationMixin,
+ VCSOperationMixin,
+ SnapStoreOperationMixin,
+ Operation,
+):
+ description = "Build a source."
+
+ @classmethod
+ def add_arguments(cls, parser):
+ super().add_arguments(parser)
+ parser.add_argument(
+ "--channel",
+ action=SnapChannelsAction,
+ metavar="SNAP=CHANNEL",
+ dest="channels",
+ default={},
+ help="install SNAP from CHANNEL",
+ )
+ parser.add_argument(
+ "--build-path", default=".", help="location of source to build."
+ )
+ parser.add_argument("name", help="name of source to build")
+
+ def __init__(self, args, parser):
+ super().__init__(args, parser)
+ self.buildd_path = os.path.join("/home/buildd", self.args.name)
+
+ def install(self):
+ logger.info("Running install phase")
+ deps = []
+ if self.args.proxy_url:
+ deps.extend(self.proxy_deps)
+ self.install_git_proxy()
+ if self.backend.supports_snapd:
+ # udev is installed explicitly to work around
+ # https://bugs.launchpad.net/snapd/+bug/1731519.
+ # Low maintenance: we can keep udevs as a dependency
+ # since it is a low-level system dependency,
+ # and since it might be broken for older versions.
+ for dep in "snapd", "fuse", "squashfuse", "udev":
+ if self.backend.is_package_available(dep):
+ deps.append(dep)
+ deps.extend(self.vcs_deps)
+ # See charmcraft.provider.CharmcraftBuilddBaseConfiguration.setup.
+ self.backend.run(["apt-get", "-y", "install"] + deps)
+ if self.backend.supports_snapd:
+ self.snap_store_set_proxy()
+ for snap_name, channel in sorted(self.args.channels.items()):
+ # sourcecraft is handled separately, since it requires --classic,
+ # which disables all sandboxing to ensure it runs with no strict
+ # confinement.
+ if snap_name != "sourcecraft":
+ self.backend.run(
+ ["snap", "install", "--channel=%s" % channel, snap_name]
+ )
+ if "sourcecraft" in self.args.channels:
+ self.backend.run(
+ [
+ "snap",
+ "install",
+ "--classic",
+ "--channel=%s" % self.args.channels["sourcecraft"],
+ "sourcecraft",
+ ]
+ )
+ else:
+ self.backend.run(["snap", "install", "--classic", "sourcecraft"])
+ # With classic confinement, the snap can access the whole system.
+ # We could build the source in /build, but we are using /home/buildd
+ # for consistency with other build types.
+ self.backend.run(["mkdir", "-p", "/home/buildd"])
+
+ def repo(self):
+ """Collect git or bzr branch."""
+ logger.info("Running repo phase...")
+ env = self.build_proxy_environment(proxy_url=self.args.proxy_url)
+ self.vcs_fetch(self.args.name, cwd="/home/buildd", env=env)
+ self.vcs_update_status(self.buildd_path)
+
+ def build(self):
+ logger.info("Running build phase...")
+ build_context_path = os.path.join(
+ "/home/buildd", self.args.name, self.args.build_path
+ )
+ check_path_escape(self.buildd_path, build_context_path)
+ env = self.build_proxy_environment(proxy_url=self.args.proxy_url)
+ args = ["sourcecraft", "pack", "-v", "--destructive-mode"]
+ self.run_build_command(args, env=env, cwd=build_context_path)
+
+ def run(self):
+ try:
+ self.install()
+ except Exception:
+ logger.exception("Install failed")
+ return RETCODE_FAILURE_INSTALL
+ try:
+ self.repo()
+ self.build()
+ except Exception:
+ logger.exception("Build failed")
+ return RETCODE_FAILURE_BUILD
+ return 0
diff --git a/lpbuildd/target/cli.py b/lpbuildd/target/cli.py
index 8d2ade2..fe7bc7b 100644
--- a/lpbuildd/target/cli.py
+++ b/lpbuildd/target/cli.py
@@ -11,6 +11,7 @@ from lpbuildd.target.build_livefs import BuildLiveFS
from lpbuildd.target.build_oci import BuildOCI
from lpbuildd.target.build_rock import BuildRock
from lpbuildd.target.build_snap import BuildSnap
+from lpbuildd.target.build_source import BuildSource
from lpbuildd.target.generate_translation_templates import (
GenerateTranslationTemplates,
)
@@ -48,6 +49,7 @@ operations = {
"build-oci": BuildOCI,
"build-charm": BuildCharm,
"build-rock": BuildRock,
+ "build-source": BuildSource,
"buildlivefs": BuildLiveFS,
"buildsnap": BuildSnap,
"generate-translation-templates": GenerateTranslationTemplates,
diff --git a/lpbuildd/target/tests/test_build_source.py b/lpbuildd/target/tests/test_build_source.py
new file mode 100644
index 0000000..c8a4848
--- /dev/null
+++ b/lpbuildd/target/tests/test_build_source.py
@@ -0,0 +1,759 @@
+import json
+import os
+import stat
+import subprocess
+from textwrap import dedent
+
+import responses
+from fixtures import FakeLogger, TempDir
+from systemfixtures import FakeFilesystem
+from testtools.matchers import AnyMatch, MatchesAll, MatchesListwise
+from testtools.testcase import TestCase
+
+from lpbuildd.target.backend import InvalidBuildFilePath
+from lpbuildd.target.build_source import (
+ RETCODE_FAILURE_BUILD,
+ RETCODE_FAILURE_INSTALL,
+)
+from lpbuildd.target.cli import parse_args
+from lpbuildd.target.tests.matchers import (
+ RanAptGet,
+ RanBuildCommand,
+ RanCommand,
+)
+from lpbuildd.target.tests.test_build_snap import FakeRevisionID, RanSnap
+from lpbuildd.tests.fakebuilder import FakeMethod
+
+
+class TestBuildSource(TestCase):
+ def test_run_build_command_no_env(self):
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--branch",
+ "lp:foo",
+ "test-image",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.run_build_command(["echo", "hello world"])
+ self.assertThat(
+ build_source.backend.run.calls,
+ MatchesListwise(
+ [
+ RanBuildCommand(
+ ["echo", "hello world"], cwd="/home/buildd/test-image"
+ ),
+ ]
+ ),
+ )
+
+ def test_run_build_command_env(self):
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--branch",
+ "lp:foo",
+ "test-image",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.run_build_command(
+ ["echo", "hello world"], env={"FOO": "bar baz"}
+ )
+ self.assertThat(
+ build_source.backend.run.calls,
+ MatchesListwise(
+ [
+ RanBuildCommand(
+ ["echo", "hello world"],
+ FOO="bar baz",
+ cwd="/home/buildd/test-image",
+ )
+ ]
+ ),
+ )
+
+ def test_install_channels(self):
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--channel=core=candidate",
+ "--channel=core18=beta",
+ "--channel=sourcecraft=edge",
+ "--branch",
+ "lp:foo",
+ "test-image",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.install()
+ self.assertThat(
+ build_source.backend.run.calls,
+ MatchesListwise(
+ [
+ RanAptGet(
+ "install", "bzr"
+ ),
+ RanSnap("install", "--channel=candidate", "core"),
+ RanSnap("install", "--channel=beta", "core18"),
+ RanSnap(
+ "install", "--classic", "--channel=edge", "sourcecraft"
+ ),
+ RanCommand(["mkdir", "-p", "/home/buildd"]),
+ ]
+ ),
+ )
+
+ def test_install_bzr(self):
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--branch",
+ "lp:foo",
+ "test-image",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.install()
+ self.assertThat(
+ build_source.backend.run.calls,
+ MatchesListwise(
+ [
+ RanAptGet(
+ "install", "bzr"
+ ),
+ RanSnap("install", "--classic", "sourcecraft"),
+ RanCommand(["mkdir", "-p", "/home/buildd"]),
+ ]
+ ),
+ )
+
+ def test_install_git(self):
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--git-repository",
+ "lp:foo",
+ "test-image",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.install()
+ self.assertThat(
+ build_source.backend.run.calls,
+ MatchesListwise(
+ [
+ RanAptGet(
+ "install", "git"
+ ),
+ RanSnap("install", "--classic", "sourcecraft"),
+ RanCommand(["mkdir", "-p", "/home/buildd"]),
+ ]
+ ),
+ )
+
+ @responses.activate
+ def test_install_snap_store_proxy(self):
+ store_assertion = dedent(
+ """\
+ type: store
+ store: store-id
+ url: http://snap-store-proxy.example
+
+ body
+ """
+ )
+
+ def respond(request):
+ return 200, {"X-Assertion-Store-Id": "store-id"}, store_assertion
+
+ responses.add_callback(
+ "GET",
+ "http://snap-store-proxy.example/v2/auth/store/assertions";,
+ callback=respond,
+ )
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--git-repository",
+ "lp:foo",
+ "--snap-store-proxy-url",
+ "http://snap-store-proxy.example/";,
+ "test-image",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.install()
+ self.assertThat(
+ build_source.backend.run.calls,
+ MatchesListwise(
+ [
+ RanAptGet(
+ "install", "git"
+ ),
+ RanCommand(
+ ["snap", "ack", "/dev/stdin"],
+ input_text=store_assertion,
+ ),
+ RanCommand(
+ ["snap", "set", "core", "proxy.store=store-id"]
+ ),
+ RanSnap("install", "--classic", "sourcecraft"),
+ RanCommand(["mkdir", "-p", "/home/buildd"]),
+ ]
+ ),
+ )
+
+ def test_install_proxy(self):
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--git-repository",
+ "lp:foo",
+ "--proxy-url",
+ "http://proxy.example:3128/";,
+ "test-image",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.bin = "/builderbin"
+ self.useFixture(FakeFilesystem()).add("/builderbin")
+ os.mkdir("/builderbin")
+ with open("/builderbin/lpbuildd-git-proxy", "w") as proxy_script:
+ proxy_script.write("proxy script\n")
+ os.fchmod(proxy_script.fileno(), 0o755)
+ build_source.install()
+ self.assertThat(
+ build_source.backend.run.calls,
+ MatchesListwise(
+ [
+ RanAptGet(
+ "install",
+ "python3",
+ "socat",
+ "git",
+ ),
+ RanSnap("install", "--classic", "sourcecraft"),
+ RanCommand(["mkdir", "-p", "/home/buildd"]),
+ ]
+ ),
+ )
+ self.assertEqual(
+ (b"proxy script\n", stat.S_IFREG | 0o755),
+ build_source.backend.backend_fs[
+ "/usr/local/bin/lpbuildd-git-proxy"
+ ],
+ )
+
+ def test_repo_bzr(self):
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--branch",
+ "lp:foo",
+ "test-image",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.backend.build_path = self.useFixture(TempDir()).path
+ build_source.backend.run = FakeRevisionID("42")
+ build_source.repo()
+ self.assertThat(
+ build_source.backend.run.calls,
+ MatchesListwise(
+ [
+ RanBuildCommand(
+ ["bzr", "branch", "lp:foo", "test-image"],
+ cwd="/home/buildd",
+ ),
+ RanBuildCommand(
+ ["bzr", "revno"],
+ cwd="/home/buildd/test-image",
+ get_output=True,
+ universal_newlines=True,
+ ),
+ ]
+ ),
+ )
+ status_path = os.path.join(build_source.backend.build_path, "status")
+ with open(status_path) as status:
+ self.assertEqual({"revision_id": "42"}, json.load(status))
+
+ def test_repo_git(self):
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--git-repository",
+ "lp:foo",
+ "test-image",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.backend.build_path = self.useFixture(TempDir()).path
+ build_source.backend.run = FakeRevisionID("0" * 40)
+ build_source.repo()
+ self.assertThat(
+ build_source.backend.run.calls,
+ MatchesListwise(
+ [
+ RanBuildCommand(
+ ["git", "clone", "-n", "lp:foo", "test-image"],
+ cwd="/home/buildd",
+ ),
+ RanBuildCommand(
+ ["git", "checkout", "-q", "HEAD"],
+ cwd="/home/buildd/test-image",
+ ),
+ RanBuildCommand(
+ [
+ "git",
+ "submodule",
+ "update",
+ "--init",
+ "--recursive",
+ ],
+ cwd="/home/buildd/test-image",
+ ),
+ RanBuildCommand(
+ ["git", "rev-parse", "HEAD^{}"],
+ cwd="/home/buildd/test-image",
+ get_output=True,
+ universal_newlines=True,
+ ),
+ ]
+ ),
+ )
+ status_path = os.path.join(build_source.backend.build_path, "status")
+ with open(status_path) as status:
+ self.assertEqual({"revision_id": "0" * 40}, json.load(status))
+
+ def test_repo_git_with_path(self):
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--git-repository",
+ "lp:foo",
+ "--git-path",
+ "next",
+ "test-image",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.backend.build_path = self.useFixture(TempDir()).path
+ build_source.backend.run = FakeRevisionID("0" * 40)
+ build_source.repo()
+ self.assertThat(
+ build_source.backend.run.calls,
+ MatchesListwise(
+ [
+ RanBuildCommand(
+ ["git", "clone", "-n", "lp:foo", "test-image"],
+ cwd="/home/buildd",
+ ),
+ RanBuildCommand(
+ ["git", "checkout", "-q", "next"],
+ cwd="/home/buildd/test-image",
+ ),
+ RanBuildCommand(
+ [
+ "git",
+ "submodule",
+ "update",
+ "--init",
+ "--recursive",
+ ],
+ cwd="/home/buildd/test-image",
+ ),
+ RanBuildCommand(
+ ["git", "rev-parse", "next^{}"],
+ cwd="/home/buildd/test-image",
+ get_output=True,
+ universal_newlines=True,
+ ),
+ ]
+ ),
+ )
+ status_path = os.path.join(build_source.backend.build_path, "status")
+ with open(status_path) as status:
+ self.assertEqual({"revision_id": "0" * 40}, json.load(status))
+
+ def test_repo_git_with_tag_path(self):
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--git-repository",
+ "lp:foo",
+ "--git-path",
+ "refs/tags/1.0",
+ "test-image",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.backend.build_path = self.useFixture(TempDir()).path
+ build_source.backend.run = FakeRevisionID("0" * 40)
+ build_source.repo()
+ self.assertThat(
+ build_source.backend.run.calls,
+ MatchesListwise(
+ [
+ RanBuildCommand(
+ ["git", "clone", "-n", "lp:foo", "test-image"],
+ cwd="/home/buildd",
+ ),
+ RanBuildCommand(
+ ["git", "checkout", "-q", "refs/tags/1.0"],
+ cwd="/home/buildd/test-image",
+ ),
+ RanBuildCommand(
+ [
+ "git",
+ "submodule",
+ "update",
+ "--init",
+ "--recursive",
+ ],
+ cwd="/home/buildd/test-image",
+ ),
+ RanBuildCommand(
+ ["git", "rev-parse", "refs/tags/1.0^{}"],
+ cwd="/home/buildd/test-image",
+ get_output=True,
+ universal_newlines=True,
+ ),
+ ]
+ ),
+ )
+ status_path = os.path.join(build_source.backend.build_path, "status")
+ with open(status_path) as status:
+ self.assertEqual({"revision_id": "0" * 40}, json.load(status))
+
+ def test_repo_proxy(self):
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--git-repository",
+ "lp:foo",
+ "--proxy-url",
+ "http://proxy.example:3128/";,
+ "test-image",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.backend.build_path = self.useFixture(TempDir()).path
+ build_source.backend.run = FakeRevisionID("0" * 40)
+ build_source.repo()
+ env = {
+ "http_proxy": "http://proxy.example:3128/";,
+ "https_proxy": "http://proxy.example:3128/";,
+ "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy",
+ "SNAPPY_STORE_NO_CDN": "1",
+ }
+ self.assertThat(
+ build_source.backend.run.calls,
+ MatchesListwise(
+ [
+ RanBuildCommand(
+ ["git", "clone", "-n", "lp:foo", "test-image"],
+ cwd="/home/buildd",
+ **env,
+ ),
+ RanBuildCommand(
+ ["git", "checkout", "-q", "HEAD"],
+ cwd="/home/buildd/test-image",
+ **env,
+ ),
+ RanBuildCommand(
+ [
+ "git",
+ "submodule",
+ "update",
+ "--init",
+ "--recursive",
+ ],
+ cwd="/home/buildd/test-image",
+ **env,
+ ),
+ RanBuildCommand(
+ ["git", "rev-parse", "HEAD^{}"],
+ cwd="/home/buildd/test-image",
+ get_output=True,
+ universal_newlines=True,
+ ),
+ ]
+ ),
+ )
+ status_path = os.path.join(build_source.backend.build_path, "status")
+ with open(status_path) as status:
+ self.assertEqual({"revision_id": "0" * 40}, json.load(status))
+
+ def test_build(self):
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--branch",
+ "lp:foo",
+ "test-image",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.backend.add_dir("/build/test-directory")
+ build_source.build()
+ self.assertThat(
+ build_source.backend.run.calls,
+ MatchesListwise(
+ [
+ RanBuildCommand(
+ ["sourcecraft", "pack", "-v", "--destructive-mode"],
+ cwd="/home/buildd/test-image/.",
+ ),
+ ]
+ ),
+ )
+
+ def test_build_with_path(self):
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--branch",
+ "lp:foo",
+ "--build-path",
+ "build-aux/",
+ "test-image",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.backend.add_dir("/build/test-directory")
+ build_source.build()
+ self.assertThat(
+ build_source.backend.run.calls,
+ MatchesListwise(
+ [
+ RanBuildCommand(
+ ["sourcecraft", "pack", "-v", "--destructive-mode"],
+ cwd="/home/buildd/test-image/build-aux/",
+ ),
+ ]
+ ),
+ )
+
+ def test_build_proxy(self):
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--branch",
+ "lp:foo",
+ "--proxy-url",
+ "http://proxy.example:3128/";,
+ "test-image",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.build()
+ env = {
+ "http_proxy": "http://proxy.example:3128/";,
+ "https_proxy": "http://proxy.example:3128/";,
+ "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy",
+ "SNAPPY_STORE_NO_CDN": "1",
+ }
+ self.assertThat(
+ build_source.backend.run.calls,
+ MatchesListwise(
+ [
+ RanBuildCommand(
+ ["sourcecraft", "pack", "-v", "--destructive-mode"],
+ cwd="/home/buildd/test-image/.",
+ **env,
+ ),
+ ]
+ ),
+ )
+
+ def test_run_succeeds(self):
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--branch",
+ "lp:foo",
+ "test-image",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.backend.build_path = self.useFixture(TempDir()).path
+ build_source.backend.run = FakeRevisionID("42")
+ self.assertEqual(0, build_source.run())
+ self.assertThat(
+ build_source.backend.run.calls,
+ MatchesAll(
+ AnyMatch(
+ RanAptGet(
+ "install", "bzr"
+ ),
+ ),
+ AnyMatch(
+ RanBuildCommand(
+ ["bzr", "branch", "lp:foo", "test-image"],
+ cwd="/home/buildd",
+ )
+ ),
+ AnyMatch(
+ RanBuildCommand(
+ ["sourcecraft", "pack", "-v", "--destructive-mode"],
+ cwd="/home/buildd/test-image/.",
+ )
+ ),
+ ),
+ )
+
+ def test_run_install_fails(self):
+ class FailInstall(FakeMethod):
+ def __call__(self, run_args, *args, **kwargs):
+ super().__call__(run_args, *args, **kwargs)
+ if run_args[0] == "apt-get":
+ raise subprocess.CalledProcessError(1, run_args)
+
+ self.useFixture(FakeLogger())
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--branch",
+ "lp:foo",
+ "test-image",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.backend.run = FailInstall()
+ self.assertEqual(RETCODE_FAILURE_INSTALL, build_source.run())
+
+ def test_run_repo_fails(self):
+ class FailRepo(FakeMethod):
+ def __call__(self, run_args, *args, **kwargs):
+ super().__call__(run_args, *args, **kwargs)
+ if run_args[:2] == ["bzr", "branch"]:
+ raise subprocess.CalledProcessError(1, run_args)
+
+ self.useFixture(FakeLogger())
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--branch",
+ "lp:foo",
+ "test-image",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.backend.run = FailRepo()
+ self.assertEqual(RETCODE_FAILURE_BUILD, build_source.run())
+
+ def test_run_build_fails(self):
+ class FailBuild(FakeMethod):
+ def __call__(self, run_args, *args, **kwargs):
+ super().__call__(run_args, *args, **kwargs)
+ if run_args[0] == "sourcecraft":
+ raise subprocess.CalledProcessError(1, run_args)
+
+ self.useFixture(FakeLogger())
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--branch",
+ "lp:foo",
+ "test-image",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.backend.build_path = self.useFixture(TempDir()).path
+ build_source.backend.run = FailBuild()
+ self.assertEqual(RETCODE_FAILURE_BUILD, build_source.run())
+
+ def test_build_with_invalid_build_path_parent(self):
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--branch",
+ "lp:foo",
+ "--build-path",
+ "../",
+ "test-image",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.backend.add_dir("/build/test-directory")
+ self.assertRaises(InvalidBuildFilePath, build_source.build)
+
+ def test_build_with_invalid_build_path_absolute(self):
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--branch",
+ "lp:foo",
+ "--build-path",
+ "/etc",
+ "test-image",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.backend.add_dir("/build/test-directory")
+ self.assertRaises(InvalidBuildFilePath, build_source.build)
+
+ def test_build_with_invalid_build_path_symlink(self):
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--branch",
+ "lp:foo",
+ "--build-path",
+ "build/",
+ "test-image",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.buildd_path = self.useFixture(TempDir()).path
+ os.symlink(
+ "/etc/hosts", os.path.join(build_source.buildd_path, "build")
+ )
+ self.assertRaises(InvalidBuildFilePath, build_source.build)
diff --git a/lpbuildd/tests/test_source.py b/lpbuildd/tests/test_source.py
new file mode 100644
index 0000000..06d9ff7
--- /dev/null
+++ b/lpbuildd/tests/test_source.py
@@ -0,0 +1,242 @@
+import base64
+import os
+
+import responses
+from fixtures import EnvironmentVariable, TempDir
+from testtools import TestCase
+from testtools.deferredruntest import AsynchronousDeferredRunTest
+from twisted.internet import defer
+
+from lpbuildd.source import SourceBuildManager, SourceBuildState
+from lpbuildd.tests.fakebuilder import FakeBuilder
+from lpbuildd.tests.matchers import HasWaitingFiles
+
+
+class MockBuildManager(SourceBuildManager):
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.commands = []
+ self.iterators = []
+
+ def runSubProcess(self, path, command, iterate=None, env=None):
+ self.commands.append([path] + command)
+ if iterate is None:
+ iterate = self.iterate
+ self.iterators.append(iterate)
+ return 0
+
+
+class TestSourceBuildManagerIteration(TestCase):
+ """Run SourceBuildManager through its iteration steps."""
+
+ run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=5)
+
+ def setUp(self):
+ super().setUp()
+ self.working_dir = self.useFixture(TempDir()).path
+ builder_dir = os.path.join(self.working_dir, "builder")
+ home_dir = os.path.join(self.working_dir, "home")
+ for dir in (builder_dir, home_dir):
+ os.mkdir(dir)
+ self.useFixture(EnvironmentVariable("HOME", home_dir))
+ self.builder = FakeBuilder(builder_dir)
+ self.buildid = "123"
+ self.buildmanager = MockBuildManager(self.builder, self.buildid)
+ self.buildmanager._cachepath = self.builder._cachepath
+
+ def getState(self):
+ """Retrieve build manager's state."""
+ return self.buildmanager._state
+
+ @defer.inlineCallbacks
+ def startBuild(self, args=None, options=None):
+ # The build manager's iterate() kicks off the consecutive states
+ # after INIT.
+ extra_args = {
+ "series": "xenial",
+ "arch_tag": "i386",
+ "name": "test-source",
+ }
+ if args is not None:
+ extra_args.update(args)
+ original_backend_name = self.buildmanager.backend_name
+ self.buildmanager.backend_name = "fake"
+ self.buildmanager.initiate({}, "chroot.tar.gz", extra_args)
+ self.buildmanager.backend_name = original_backend_name
+
+ # Skip states that are done in DebianBuildManager to the state
+ # directly before BUILD_SOURCE.
+ self.buildmanager._state = SourceBuildState.UPDATE
+
+ # BUILD_SOURCE: Run the builder's payload to build the source.
+ yield self.buildmanager.iterate(0)
+ self.assertEqual(SourceBuildState.BUILD_SOURCE, self.getState())
+ expected_command = [
+ "sharepath/bin/in-target",
+ "in-target",
+ "build-source",
+ "--backend=lxd",
+ "--series=xenial",
+ "--arch=i386",
+ self.buildid,
+ ]
+ if options is not None:
+ expected_command.extend(options)
+ expected_command.append("test-source")
+ self.assertEqual(expected_command, self.buildmanager.commands[-1])
+ self.assertEqual(
+ self.buildmanager.iterate, self.buildmanager.iterators[-1]
+ )
+ self.assertFalse(self.builder.wasCalled("chrootFail"))
+
+ def test_status(self):
+ # The build manager returns saved status information on request.
+ self.assertEqual({}, self.buildmanager.status())
+ status_path = os.path.join(
+ self.working_dir, "home", "build-%s" % self.buildid, "status"
+ )
+ os.makedirs(os.path.dirname(status_path))
+ with open(status_path, "w") as status_file:
+ status_file.write('{"revision_id": "foo"}')
+ self.assertEqual({"revision_id": "foo"}, self.buildmanager.status())
+
+ @defer.inlineCallbacks
+ def test_iterate(self):
+ # The build manager iterates a normal build from start to finish.
+ args = {
+ "git_repository": "https://git.launchpad.dev/~example/+git/source";,
+ "git_path": "master",
+ }
+ expected_options = [
+ "--git-repository",
+ "https://git.launchpad.dev/~example/+git/source";,
+ "--git-path",
+ "master",
+ ]
+ yield self.startBuild(args, expected_options)
+
+ log_path = os.path.join(self.buildmanager._cachepath, "buildlog")
+ with open(log_path, "w") as log:
+ log.write("I am a build log.")
+
+ self.buildmanager.backend.add_file(
+ "/home/buildd/test-source/test-source_0_all.source", b"I am sourceing."
+ )
+
+ # After building the package, reap processes.
+ yield self.buildmanager.iterate(0)
+ expected_command = [
+ "sharepath/bin/in-target",
+ "in-target",
+ "scan-for-processes",
+ "--backend=lxd",
+ "--series=xenial",
+ "--arch=i386",
+ self.buildid,
+ ]
+
+ self.assertEqual(SourceBuildState.BUILD_SOURCE, self.getState())
+ self.assertEqual(expected_command, self.buildmanager.commands[-1])
+ self.assertNotEqual(
+ self.buildmanager.iterate, self.buildmanager.iterators[-1]
+ )
+ self.assertFalse(self.builder.wasCalled("buildFail"))
+ self.assertThat(
+ self.builder,
+ HasWaitingFiles.byEquality(
+ {
+ "test-source_0_all.source": b"I am sourceing.",
+ }
+ ),
+ )
+
+ # Control returns to the DebianBuildManager in the UMOUNT state.
+ self.buildmanager.iterateReap(self.getState(), 0)
+ expected_command = [
+ "sharepath/bin/in-target",
+ "in-target",
+ "umount-chroot",
+ "--backend=lxd",
+ "--series=xenial",
+ "--arch=i386",
+ self.buildid,
+ ]
+ self.assertEqual(SourceBuildState.UMOUNT, self.getState())
+ self.assertEqual(expected_command, self.buildmanager.commands[-1])
+ self.assertEqual(
+ self.buildmanager.iterate, self.buildmanager.iterators[-1]
+ )
+ self.assertFalse(self.builder.wasCalled("buildFail"))
+
+ @defer.inlineCallbacks
+ def test_iterate_build_path(self):
+ # The build manager iterates a build using build_path from start to
+ # finish.
+ args = {
+ "git_repository": "https://git.launchpad.dev/~example/+git/source";,
+ "git_path": "master",
+ "build_path": "source",
+ }
+ expected_options = [
+ "--git-repository",
+ "https://git.launchpad.dev/~example/+git/source";,
+ "--git-path",
+ "master",
+ "--build-path",
+ "source",
+ ]
+ yield self.startBuild(args, expected_options)
+
+ log_path = os.path.join(self.buildmanager._cachepath, "buildlog")
+ with open(log_path, "w") as log:
+ log.write("I am a build log.")
+
+ self.buildmanager.backend.add_file(
+ "/home/buildd/test-source/source/test-source_0_all.source",
+ b"I am sourceing.",
+ )
+
+ # After building the package, reap processes.
+ yield self.buildmanager.iterate(0)
+ expected_command = [
+ "sharepath/bin/in-target",
+ "in-target",
+ "scan-for-processes",
+ "--backend=lxd",
+ "--series=xenial",
+ "--arch=i386",
+ self.buildid,
+ ]
+
+ self.assertEqual(SourceBuildState.BUILD_SOURCE, self.getState())
+ self.assertEqual(expected_command, self.buildmanager.commands[-1])
+ self.assertNotEqual(
+ self.buildmanager.iterate, self.buildmanager.iterators[-1]
+ )
+ self.assertFalse(self.builder.wasCalled("buildFail"))
+ self.assertThat(
+ self.builder,
+ HasWaitingFiles.byEquality(
+ {
+ "test-source_0_all.source": b"I am sourceing.",
+ }
+ ),
+ )
+
+ # Control returns to the DebianBuildManager in the UMOUNT state.
+ self.buildmanager.iterateReap(self.getState(), 0)
+ expected_command = [
+ "sharepath/bin/in-target",
+ "in-target",
+ "umount-chroot",
+ "--backend=lxd",
+ "--series=xenial",
+ "--arch=i386",
+ self.buildid,
+ ]
+ self.assertEqual(SourceBuildState.UMOUNT, self.getState())
+ self.assertEqual(expected_command, self.buildmanager.commands[-1])
+ self.assertEqual(
+ self.buildmanager.iterate, self.buildmanager.iterators[-1]
+ )
+ self.assertFalse(self.builder.wasCalled("buildFail"))
