launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #23805
[Merge] lp:~twom/launchpad-buildd/initial-docker-build-support into lp:launchpad-buildd
Tom Wardill has proposed merging lp:~twom/launchpad-buildd/initial-docker-build-support into lp:launchpad-buildd.
Commit message:
Add initial docker build support
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~twom/launchpad-buildd/initial-docker-build-support/+merge/369775
Add a builder for docker, creating an image following a supplied Dockerfile.
Save the image, extract it and then tar each component layer individually for returning/caching in launchpad.
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~twom/launchpad-buildd/initial-docker-build-support into lp:launchpad-buildd.
=== modified file 'debian/changelog'
--- debian/changelog 2019-06-18 16:54:33 +0000
+++ debian/changelog 2019-07-05 15:18:01 +0000
@@ -1,3 +1,9 @@
+launchpad-buildd (177) UNRELEASED; urgency=medium
+
++ * Prototype Docker image building support.
+
+ -- Colin Watson <cjwatson@xxxxxxxxxx> Wed, 05 Jun 2019 15:06:54 +0100
+
launchpad-buildd (176) xenial; urgency=medium
* Don't rely on /CurrentlyBuilding existing in base images.
@@ -726,7 +732,7 @@
memory at once (LP: #1227086).
[ Adam Conrad ]
- * Tidy up log formatting of the "Already reaped..." message.
+ * Tidy up log formatting of the "Already reaped..." message.
-- Colin Watson <cjwatson@xxxxxxxxxx> Fri, 27 Sep 2013 13:08:59 +0100
@@ -911,7 +917,7 @@
launchpad-buildd (98) hardy; urgency=low
* Add launchpad-buildd dependency on python-apt, as an accomodation for it
- being only a Recommends but actually required by python-debian.
+ being only a Recommends but actually required by python-debian.
LP: #890834
-- Martin Pool <mbp@xxxxxxxxxxxxx> Wed, 16 Nov 2011 10:28:48 +1100
@@ -965,7 +971,7 @@
launchpad-buildd (90) hardy; urgency=low
- * debhelper is a Build-Depends because it is needed to run 'clean'.
+ * debhelper is a Build-Depends because it is needed to run 'clean'.
* python-lpbuildd conflicts with launchpad-buildd << 88.
* Add and adjust build-arch, binary-arch, build-indep to match policy.
* Complies with stardards version 3.9.2.
=== modified file 'lpbuildd/buildd-slave.tac'
--- lpbuildd/buildd-slave.tac 2019-02-12 10:35:12 +0000
+++ lpbuildd/buildd-slave.tac 2019-07-05 15:18:01 +0000
@@ -23,6 +23,7 @@
from lpbuildd.binarypackage import BinaryPackageBuildManager
from lpbuildd.builder import XMLRPCBuilder
+from lpbuildd.docker import DockerBuildManager
from lpbuildd.livefs import LiveFilesystemBuildManager
from lpbuildd.log import RotatableFileLogObserver
from lpbuildd.snap import SnapBuildManager
@@ -45,6 +46,7 @@
TranslationTemplatesBuildManager, 'translation-templates')
builder.registerManager(LiveFilesystemBuildManager, "livefs")
builder.registerManager(SnapBuildManager, "snap")
+builder.registerManager(DockerBuildManager, "docker")
application = service.Application('Builder')
application.addComponent(
=== added file 'lpbuildd/docker.py'
--- lpbuildd/docker.py 1970-01-01 00:00:00 +0000
+++ lpbuildd/docker.py 2019-07-05 15:18:01 +0000
@@ -0,0 +1,167 @@
+# Copyright 2019 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from __future__ import print_function
+
+__metaclass__ = type
+
+import base64
+import json
+import os
+import tempfile
+
+from six.moves.configparser import (
+ NoOptionError,
+ NoSectionError,
+ )
+from six.moves.urllib.error import (
+ HTTPError,
+ URLError,
+ )
+from six.moves.urllib.parse import urlparse
+from six.moves.urllib.request import (
+ Request,
+ urlopen,
+ )
+from twisted.application import strports
+
+from lpbuildd.debian import (
+ DebianBuildManager,
+ DebianBuildState,
+ )
+from lpbuildd.snap import SnapProxyFactory
+
+
+RETCODE_SUCCESS = 0
+RETCODE_FAILURE_INSTALL = 200
+RETCODE_FAILURE_BUILD = 201
+
+
+class DockerBuildState(DebianBuildState):
+ BUILD_DOCKER = "BUILD_DOCKER"
+
+
+class DockerBuildManager(DebianBuildManager):
+ """Build a snap."""
+
+ backend_name = "lxd"
+ initial_build_state = DockerBuildState.BUILD_DOCKER
+
+ @property
+ def needs_sanitized_logs(self):
+ return True
+
+ def initiate(self, files, chroot, extra_args):
+ """Initiate a build with a given set of files and chroot."""
+ self.name = extra_args["name"]
+ self.branch = extra_args.get("branch")
+ self.git_repository = extra_args.get("git_repository")
+ self.git_path = extra_args.get("git_path")
+ self.file = extra_args.get("file")
+ self.proxy_url = extra_args.get("proxy_url")
+ self.revocation_endpoint = extra_args.get("revocation_endpoint")
+ self.proxy_service = None
+
+ super(DockerBuildManager, self).initiate(files, chroot, extra_args)
+
+ def startProxy(self):
+ """Start the local snap proxy, if necessary."""
+ if not self.proxy_url:
+ return []
+ proxy_port = self._builder._config.get("snapmanager", "proxyport")
+ proxy_factory = SnapProxyFactory(self, self.proxy_url, timeout=60)
+ self.proxy_service = strports.service(proxy_port, proxy_factory)
+ self.proxy_service.setServiceParent(self._builder.service)
+ if self.backend_name == "lxd":
+ proxy_host = self.backend.ipv4_network.ip
+ else:
+ proxy_host = "localhost"
+ return ["--proxy-url", "http://{}:{}/".format(proxy_host, proxy_port)]
+
+ def stopProxy(self):
+ """Stop the local snap proxy, if necessary."""
+ if self.proxy_service is None:
+ return
+ self.proxy_service.disownServiceParent()
+ self.proxy_service = None
+
+ def revokeProxyToken(self):
+ """Revoke builder proxy token."""
+ if not self.revocation_endpoint:
+ return
+ self._builder.log("Revoking proxy token...\n")
+ url = urlparse(self.proxy_url)
+ auth = "{}:{}".format(url.username, url.password)
+ headers = {
+ "Authorization": "Basic {}".format(base64.b64encode(auth))
+ }
+ req = Request(self.revocation_endpoint, None, headers)
+ req.get_method = lambda: "DELETE"
+ try:
+ urlopen(req)
+ except (HTTPError, URLError) as e:
+ self._builder.log(
+ "Unable to revoke token for %s: %s" % (url.username, e))
+
+ def doRunBuild(self):
+ """Run the process to build the snap."""
+ args = []
+ args.extend(self.startProxy())
+ if self.revocation_endpoint:
+ args.extend(["--revocation-endpoint", self.revocation_endpoint])
+ if self.branch is not None:
+ args.extend(["--branch", self.branch])
+ if self.git_repository is not None:
+ args.extend(["--git-repository", self.git_repository])
+ if self.git_path is not None:
+ args.extend(["--git-path", self.git_path])
+ if self.file is not None:
+ args.extend(["--file", self.file])
+ try:
+ snap_store_proxy_url = self._builder._config.get(
+ "proxy", "snapstore")
+ args.extend(["--snap-store-proxy-url", snap_store_proxy_url])
+ except (NoSectionError, NoOptionError):
+ pass
+ args.append(self.name)
+ self.runTargetSubProcess("build-docker", *args)
+
+ def iterate_BUILD_DOCKER(self, retcode):
+ """Finished building the Docker image."""
+ self.stopProxy()
+ self.revokeProxyToken()
+ if retcode == RETCODE_SUCCESS:
+ print("Returning build status: OK")
+ return self.deferGatherResults()
+ elif (retcode >= RETCODE_FAILURE_INSTALL and
+ retcode <= RETCODE_FAILURE_BUILD):
+ if not self.alreadyfailed:
+ self._builder.buildFail()
+ print("Returning build status: Build failed.")
+ self.alreadyfailed = True
+ else:
+ if not self.alreadyfailed:
+ self._builder.builderFail()
+ print("Returning build status: Builder failed.")
+ self.alreadyfailed = True
+ self.doReapProcesses(self._state)
+
+ def iterateReap_BUILD_DOCKER(self, retcode):
+ """Finished reaping after building the Docker image."""
+ self._state = DebianBuildState.UMOUNT
+ self.doUnmounting()
+
+ def gatherResults(self):
+ """Gather the results of the build and add them to the file cache."""
+ self.addWaitingFileFromBackend('/build/manifest.json')
+ with tempfile.NamedTemporaryFile() as manifest_path:
+ self.backend.copy_out('/build/manifest.json', manifest_path.name)
+ with open(manifest_path.name) as manifest_fp:
+ manifest = json.load(manifest_fp)
+
+ for section in manifest:
+ layers = section['Layers']
+ for layer in layers:
+ layer_name = layer.split('/')[0]
+ layer_path = os.path.join('/build/', layer_name + '.tar')
+ self.addWaitingFileFromBackend(layer_path)
=== added file 'lpbuildd/target/build_docker.py'
--- lpbuildd/target/build_docker.py 1970-01-01 00:00:00 +0000
+++ lpbuildd/target/build_docker.py 2019-07-05 15:18:01 +0000
@@ -0,0 +1,143 @@
+# Copyright 2019 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from __future__ import print_function
+
+__metaclass__ = type
+
+from collections import OrderedDict
+import logging
+import os.path
+import sys
+
+from lpbuildd.target.operation import Operation
+from lpbuildd.target.snapstore import SnapStoreOperationMixin
+from lpbuildd.target.vcs import VCSOperationMixin
+
+
+RETCODE_FAILURE_INSTALL = 200
+RETCODE_FAILURE_BUILD = 201
+
+
+logger = logging.getLogger(__name__)
+
+
+class BuildDocker(VCSOperationMixin, SnapStoreOperationMixin, Operation):
+
+ description = "Build a Docker image."
+
+ @classmethod
+ def add_arguments(cls, parser):
+ super(BuildDocker, cls).add_arguments(parser)
+ parser.add_argument("--proxy-url", help="builder proxy url")
+ parser.add_argument(
+ "--revocation-endpoint",
+ help="builder proxy token revocation endpoint")
+ parser.add_argument("--file", help="path to Dockerfile in branch")
+ parser.add_argument("name", help="name of snap to build")
+
+ def __init__(self, args, parser):
+ super(BuildDocker, self).__init__(args, parser)
+ self.bin = os.path.dirname(sys.argv[0])
+
+ def run_build_command(self, args, env=None, **kwargs):
+ """Run a build command in the target.
+
+ :param args: the command and arguments to run.
+ :param env: dictionary of additional environment variables to set.
+ :param kwargs: any other keyword arguments to pass to Backend.run.
+ """
+ full_env = OrderedDict()
+ full_env["LANG"] = "C.UTF-8"
+ full_env["SHELL"] = "/bin/sh"
+ if env:
+ full_env.update(env)
+ return self.backend.run(args, env=full_env, **kwargs)
+
+ def install(self):
+ logger.info("Running install phase...")
+ deps = []
+ if self.args.backend == "lxd":
+ # udev is installed explicitly to work around
+ # https://bugs.launchpad.net/snapd/+bug/1731519.
+ for dep in "snapd", "fuse", "squashfuse", "udev":
+ if self.backend.is_package_available(dep):
+ deps.append(dep)
+ deps.extend(self.vcs_deps)
+ if self.args.proxy_url:
+ deps.extend(["python3", "socat"])
+ self.backend.run(["apt-get", "-y", "install"] + deps)
+ if self.args.backend in ("lxd", "fake"):
+ self.snap_store_set_proxy()
+ self.backend.run(["snap", "install", "docker"])
+ if self.args.proxy_url:
+ self.backend.copy_in(
+ os.path.join(self.bin, "snap-git-proxy"),
+ "/usr/local/bin/snap-git-proxy")
+ # The docker snap can't see /build, so we have to do our work under
+ # /home/buildd instead. Make sure it exists.
+ self.backend.run(["mkdir", "-p", "/home/buildd"])
+
+ def repo(self):
+ """Collect git or bzr branch."""
+ logger.info("Running repo phase...")
+ env = OrderedDict()
+ if self.args.proxy_url:
+ env["http_proxy"] = self.args.proxy_url
+ env["https_proxy"] = self.args.proxy_url
+ env["GIT_PROXY_COMMAND"] = "/usr/local/bin/snap-git-proxy"
+ self.vcs_fetch(self.args.name, cwd="/home/buildd", env=env)
+
+ def build(self):
+ logger.info("Running build phase...")
+ args = ["docker", "build", "--no-cache"]
+ if self.args.proxy_url:
+ for var in ("http_proxy", "https_proxy"):
+ args.extend(
+ ["--build-arg", "{}={}".format(var, self.args.proxy_url)])
+ args.extend(["--tag", self.args.name])
+ if self.args.file is not None:
+ args.extend(["--file", self.args.file])
+ args.append(os.path.join("/home/buildd", self.args.name))
+ self.run_build_command(args)
+
+ # Make extraction directy
+ self.backend.run(["mkdir", "-p", "/home/buildd/{}-extract".format(
+ self.args.name)])
+
+ # save the newly built image
+ docker_save = "docker save {name} > /build/{name}.tar".format(
+ name=self.args.name)
+ save_args = ["/bin/bash", "-c", docker_save]
+ self.run_build_command(save_args)
+
+ # extract the saved image
+ extract_args = [
+ "tar", "-xf", "/build/{name}.tar".format(name=self.args.name),
+ "-C", "/build/"
+ ]
+ self.run_build_command(extract_args)
+
+ # Tar each layer separately
+ build_dir_contents = self.backend.listdir('/build')
+ for content in build_dir_contents:
+ content_path = os.path.join('/build/', content)
+ if not self.backend.isdir(content_path):
+ continue
+ tar_path = '/build/{}.tar'.format(content)
+ tar_args = ['tar', '-cvf', tar_path, content_path]
+ self.run_build_command(tar_args)
+
+ def run(self):
+ try:
+ self.install()
+ except Exception:
+ logger.exception('Install failed')
+ return RETCODE_FAILURE_INSTALL
+ try:
+ self.repo()
+ self.build()
+ except Exception:
+ logger.exception('Build failed')
+ return RETCODE_FAILURE_BUILD
+ return 0
=== modified file 'lpbuildd/target/cli.py'
--- lpbuildd/target/cli.py 2017-09-08 15:57:18 +0000
+++ lpbuildd/target/cli.py 2019-07-05 15:18:01 +0000
@@ -14,6 +14,7 @@
OverrideSourcesList,
Update,
)
+from lpbuildd.target.build_docker import BuildDocker
from lpbuildd.target.build_livefs import BuildLiveFS
from lpbuildd.target.build_snap import BuildSnap
from lpbuildd.target.generate_translation_templates import (
@@ -49,6 +50,7 @@
operations = {
"add-trusted-keys": AddTrustedKeys,
+ "build-docker": BuildDocker,
"buildlivefs": BuildLiveFS,
"buildsnap": BuildSnap,
"generate-translation-templates": GenerateTranslationTemplates,
=== added file 'lpbuildd/target/tests/test_build_docker.py'
--- lpbuildd/target/tests/test_build_docker.py 1970-01-01 00:00:00 +0000
+++ lpbuildd/target/tests/test_build_docker.py 2019-07-05 15:18:01 +0000
@@ -0,0 +1,433 @@
+# Copyright 2019 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+__metaclass__ = type
+
+import os.path
+import stat
+import subprocess
+from textwrap import dedent
+
+from fixtures import (
+ FakeLogger,
+ TempDir,
+ )
+import responses
+from systemfixtures import FakeFilesystem
+from testtools import TestCase
+from testtools.matchers import (
+ AnyMatch,
+ Equals,
+ Is,
+ MatchesAll,
+ MatchesDict,
+ MatchesListwise,
+ )
+
+from lpbuildd.target.build_docker import (
+ RETCODE_FAILURE_BUILD,
+ RETCODE_FAILURE_INSTALL,
+ )
+from lpbuildd.target.cli import parse_args
+from lpbuildd.tests.fakebuilder import FakeMethod
+
+
+class RanCommand(MatchesListwise):
+
+ def __init__(self, args, echo=None, cwd=None, input_text=None,
+ get_output=None, **env):
+ kwargs_matcher = {}
+ if echo is not None:
+ kwargs_matcher["echo"] = Is(echo)
+ if cwd:
+ kwargs_matcher["cwd"] = Equals(cwd)
+ if input_text:
+ kwargs_matcher["input_text"] = Equals(input_text)
+ if get_output is not None:
+ kwargs_matcher["get_output"] = Is(get_output)
+ if env:
+ kwargs_matcher["env"] = MatchesDict(
+ {key: Equals(value) for key, value in env.items()})
+ super(RanCommand, self).__init__(
+ [Equals((args,)), MatchesDict(kwargs_matcher)])
+
+
+class RanAptGet(RanCommand):
+
+ def __init__(self, *args):
+ super(RanAptGet, self).__init__(["apt-get", "-y"] + list(args))
+
+
+class RanSnap(RanCommand):
+
+ def __init__(self, *args, **kwargs):
+ super(RanSnap, self).__init__(["snap"] + list(args), **kwargs)
+
+
+class RanBuildCommand(RanCommand):
+
+ def __init__(self, args, **kwargs):
+ kwargs.setdefault("LANG", "C.UTF-8")
+ kwargs.setdefault("SHELL", "/bin/sh")
+ super(RanBuildCommand, self).__init__(args, **kwargs)
+
+
+class TestBuildDocker(TestCase):
+
+ def test_run_build_command_no_env(self):
+ args = [
+ "build-docker",
+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+ "--branch", "lp:foo", "test-image",
+ ]
+ build_docker = parse_args(args=args).operation
+ build_docker.run_build_command(["echo", "hello world"])
+ self.assertThat(build_docker.backend.run.calls, MatchesListwise([
+ RanBuildCommand(["echo", "hello world"]),
+ ]))
+
+ def test_run_build_command_env(self):
+ args = [
+ "build-docker",
+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+ "--branch", "lp:foo", "test-image",
+ ]
+ build_docker = parse_args(args=args).operation
+ build_docker.run_build_command(
+ ["echo", "hello world"], env={"FOO": "bar baz"})
+ self.assertThat(build_docker.backend.run.calls, MatchesListwise([
+ RanBuildCommand(["echo", "hello world"], FOO="bar baz"),
+ ]))
+
+ def test_install_bzr(self):
+ args = [
+ "build-docker",
+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+ "--branch", "lp:foo", "test-image"
+ ]
+ build_docker = parse_args(args=args).operation
+ build_docker.install()
+ self.assertThat(build_docker.backend.run.calls, MatchesListwise([
+ RanAptGet("install", "bzr"),
+ RanSnap("install", "docker"),
+ RanCommand(["mkdir", "-p", "/home/buildd"]),
+ ]))
+
+ def test_install_git(self):
+ args = [
+ "build-docker",
+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+ "--git-repository", "lp:foo", "test-image"
+ ]
+ build_docker = parse_args(args=args).operation
+ build_docker.install()
+ self.assertThat(build_docker.backend.run.calls, MatchesListwise([
+ RanAptGet("install", "git"),
+ RanSnap("install", "docker"),
+ RanCommand(["mkdir", "-p", "/home/buildd"]),
+ ]))
+
+ @responses.activate
+ def test_install_snap_store_proxy(self):
+ store_assertion = dedent("""\
+ type: store
+ store: store-id
+ url: http://snap-store-proxy.example
+
+ body
+ """)
+
+ def respond(request):
+ return 200, {"X-Assertion-Store-Id": "store-id"}, store_assertion
+
+ responses.add_callback(
+ "GET", "http://snap-store-proxy.example/v2/auth/store/assertions";,
+ callback=respond)
+ args = [
+ "build-docker",
+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+ "--git-repository", "lp:foo",
+ "--snap-store-proxy-url", "http://snap-store-proxy.example/";,
+ "test-image",
+ ]
+ build_snap = parse_args(args=args).operation
+ build_snap.install()
+ self.assertThat(build_snap.backend.run.calls, MatchesListwise([
+ RanAptGet("install", "git"),
+ RanSnap("ack", "/dev/stdin", input_text=store_assertion),
+ RanSnap("set", "core", "proxy.store=store-id"),
+ RanSnap("install", "docker"),
+ RanCommand(["mkdir", "-p", "/home/buildd"]),
+ ]))
+
+ def test_install_proxy(self):
+ args = [
+ "build-docker",
+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+ "--git-repository", "lp:foo",
+ "--proxy-url", "http://proxy.example:3128/";,
+ "test-image",
+ ]
+ build_docker = parse_args(args=args).operation
+ build_docker.bin = "/builderbin"
+ self.useFixture(FakeFilesystem()).add("/builderbin")
+ os.mkdir("/builderbin")
+ with open("/builderbin/snap-git-proxy", "w") as proxy_script:
+ proxy_script.write("proxy script\n")
+ os.fchmod(proxy_script.fileno(), 0o755)
+ build_docker.install()
+ self.assertThat(build_docker.backend.run.calls, MatchesListwise([
+ RanAptGet("install", "git", "python3", "socat"),
+ RanSnap("install", "docker"),
+ RanCommand(["mkdir", "-p", "/home/buildd"]),
+ ]))
+ self.assertEqual(
+ (b"proxy script\n", stat.S_IFREG | 0o755),
+ build_docker.backend.backend_fs["/usr/local/bin/snap-git-proxy"])
+
+ def test_repo_bzr(self):
+ args = [
+ "build-docker",
+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+ "--branch", "lp:foo", "test-image",
+ ]
+ build_docker = parse_args(args=args).operation
+ build_docker.backend.build_path = self.useFixture(TempDir()).path
+ build_docker.backend.run = FakeMethod()
+ build_docker.repo()
+ self.assertThat(build_docker.backend.run.calls, MatchesListwise([
+ RanBuildCommand(
+ ["bzr", "branch", "lp:foo", "test-image"], cwd="/home/buildd"),
+ ]))
+
+ def test_repo_git(self):
+ args = [
+ "build-docker",
+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+ "--git-repository", "lp:foo", "test-image",
+ ]
+ build_docker = parse_args(args=args).operation
+ build_docker.backend.build_path = self.useFixture(TempDir()).path
+ build_docker.backend.run = FakeMethod()
+ build_docker.repo()
+ self.assertThat(build_docker.backend.run.calls, MatchesListwise([
+ RanBuildCommand(
+ ["git", "clone", "lp:foo", "test-image"], cwd="/home/buildd"),
+ RanBuildCommand(
+ ["git", "submodule", "update", "--init", "--recursive"],
+ cwd="/home/buildd/test-image"),
+ ]))
+
+ def test_repo_git_with_path(self):
+ args = [
+ "build-docker",
+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+ "--git-repository", "lp:foo", "--git-path", "next", "test-image",
+ ]
+ build_docker = parse_args(args=args).operation
+ build_docker.backend.build_path = self.useFixture(TempDir()).path
+ build_docker.backend.run = FakeMethod()
+ build_docker.repo()
+ self.assertThat(build_docker.backend.run.calls, MatchesListwise([
+ RanBuildCommand(
+ ["git", "clone", "-b", "next", "lp:foo", "test-image"],
+ cwd="/home/buildd"),
+ RanBuildCommand(
+ ["git", "submodule", "update", "--init", "--recursive"],
+ cwd="/home/buildd/test-image"),
+ ]))
+
+ def test_repo_git_with_tag_path(self):
+ args = [
+ "build-docker",
+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+ "--git-repository", "lp:foo", "--git-path", "refs/tags/1.0",
+ "test-image",
+ ]
+ build_docker = parse_args(args=args).operation
+ build_docker.backend.build_path = self.useFixture(TempDir()).path
+ build_docker.backend.run = FakeMethod()
+ build_docker.repo()
+ self.assertThat(build_docker.backend.run.calls, MatchesListwise([
+ RanBuildCommand(
+ ["git", "clone", "-b", "1.0", "lp:foo", "test-image"],
+ cwd="/home/buildd"),
+ RanBuildCommand(
+ ["git", "submodule", "update", "--init", "--recursive"],
+ cwd="/home/buildd/test-image"),
+ ]))
+
+ def test_repo_proxy(self):
+ args = [
+ "build-docker",
+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+ "--git-repository", "lp:foo",
+ "--proxy-url", "http://proxy.example:3128/";,
+ "test-image",
+ ]
+ build_docker = parse_args(args=args).operation
+ build_docker.backend.build_path = self.useFixture(TempDir()).path
+ build_docker.backend.run = FakeMethod()
+ build_docker.repo()
+ env = {
+ "http_proxy": "http://proxy.example:3128/";,
+ "https_proxy": "http://proxy.example:3128/";,
+ "GIT_PROXY_COMMAND": "/usr/local/bin/snap-git-proxy",
+ }
+ self.assertThat(build_docker.backend.run.calls, MatchesListwise([
+ RanBuildCommand(
+ ["git", "clone", "lp:foo", "test-image"],
+ cwd="/home/buildd", **env),
+ RanBuildCommand(
+ ["git", "submodule", "update", "--init", "--recursive"],
+ cwd="/home/buildd/test-image", **env),
+ ]))
+
+ def test_build(self):
+ args = [
+ "build-docker",
+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+ "--branch", "lp:foo", "test-image",
+ ]
+ build_docker = parse_args(args=args).operation
+ build_docker.backend.add_dir('/build/test-directory')
+ build_docker.build()
+ self.assertThat(build_docker.backend.run.calls, MatchesListwise([
+ RanBuildCommand(
+ ["docker", "build", "--no-cache", "--tag", "test-image",
+ "/home/buildd/test-image"]),
+ RanCommand(["mkdir", "-p", "/home/buildd/test-image-extract"]),
+ RanBuildCommand([
+ '/bin/bash', '-c',
+ 'docker save test-image > /build/test-image.tar']),
+ RanBuildCommand([
+ 'tar', '-xf', '/build/test-image.tar', '-C', '/build/']),
+ RanBuildCommand([
+ 'tar', '-cvf', '/build/test-directory.tar',
+ '/build/test-directory']),
+ ]))
+
+ def test_build_with_file(self):
+ args = [
+ "build-docker",
+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+ "--branch", "lp:foo", "--file", "build-aux/Dockerfile",
+ "test-image",
+ ]
+ build_docker = parse_args(args=args).operation
+ build_docker.backend.add_dir('/build/test-directory')
+ build_docker.build()
+ self.assertThat(build_docker.backend.run.calls, MatchesListwise([
+ RanBuildCommand(
+ ["docker", "build", "--no-cache", "--tag", "test-image",
+ "--file", "build-aux/Dockerfile", "/home/buildd/test-image"]),
+ RanCommand(["mkdir", "-p", "/home/buildd/test-image-extract"]),
+ RanBuildCommand([
+ '/bin/bash', '-c',
+ 'docker save test-image > /build/test-image.tar']),
+ RanBuildCommand([
+ 'tar', '-xf', '/build/test-image.tar', '-C', '/build/']),
+ RanBuildCommand([
+ 'tar', '-cvf', '/build/test-directory.tar',
+ '/build/test-directory']),
+ ]))
+
+ def test_build_proxy(self):
+ args = [
+ "build-docker",
+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+ "--branch", "lp:foo", "--proxy-url", "http://proxy.example:3128/";,
+ "test-image",
+ ]
+ build_docker = parse_args(args=args).operation
+ build_docker.backend.add_dir('/build/test-directory')
+ build_docker.build()
+ self.assertThat(build_docker.backend.run.calls, MatchesListwise([
+ RanBuildCommand(
+ ["docker", "build", "--no-cache",
+ "--build-arg", "http_proxy=http://proxy.example:3128/";,
+ "--build-arg", "https_proxy=http://proxy.example:3128/";,
+ "--tag", "test-image", "/home/buildd/test-image"]),
+ RanCommand(["mkdir", "-p", "/home/buildd/test-image-extract"]),
+ RanBuildCommand([
+ '/bin/bash', '-c',
+ 'docker save test-image > /build/test-image.tar']),
+ RanBuildCommand([
+ 'tar', '-xf', '/build/test-image.tar', '-C', '/build/']),
+ RanBuildCommand([
+ 'tar', '-cvf', '/build/test-directory.tar',
+ '/build/test-directory']),
+ ]))
+
+ def test_run_succeeds(self):
+ args = [
+ "build-docker",
+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+ "--branch", "lp:foo", "test-image",
+ ]
+ build_docker = parse_args(args=args).operation
+ build_docker.backend.build_path = self.useFixture(TempDir()).path
+ build_docker.backend.run = FakeMethod()
+ self.assertEqual(0, build_docker.run())
+ self.assertThat(build_docker.backend.run.calls, MatchesAll(
+ AnyMatch(RanAptGet("install", "bzr")),
+ AnyMatch(RanSnap("install", "docker")),
+ AnyMatch(RanBuildCommand(
+ ["bzr", "branch", "lp:foo", "test-image"],
+ cwd="/home/buildd")),
+ AnyMatch(RanBuildCommand(
+ ["docker", "build", "--no-cache", "--tag", "test-image",
+ "/home/buildd/test-image"])),
+ ))
+
+ def test_run_install_fails(self):
+ class FailInstall(FakeMethod):
+ def __call__(self, run_args, *args, **kwargs):
+ super(FailInstall, self).__call__(run_args, *args, **kwargs)
+ if run_args[0] == "apt-get":
+ raise subprocess.CalledProcessError(1, run_args)
+
+ self.useFixture(FakeLogger())
+ args = [
+ "build-docker",
+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+ "--branch", "lp:foo", "test-image",
+ ]
+ build_docker = parse_args(args=args).operation
+ build_docker.backend.run = FailInstall()
+ self.assertEqual(RETCODE_FAILURE_INSTALL, build_docker.run())
+
+ def test_run_repo_fails(self):
+ class FailRepo(FakeMethod):
+ def __call__(self, run_args, *args, **kwargs):
+ super(FailRepo, self).__call__(run_args, *args, **kwargs)
+ if run_args[:2] == ["bzr", "branch"]:
+ raise subprocess.CalledProcessError(1, run_args)
+
+ self.useFixture(FakeLogger())
+ args = [
+ "build-docker",
+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+ "--branch", "lp:foo", "test-image",
+ ]
+ build_docker = parse_args(args=args).operation
+ build_docker.backend.run = FailRepo()
+ self.assertEqual(RETCODE_FAILURE_BUILD, build_docker.run())
+
+ def test_run_build_fails(self):
+ class FailBuild(FakeMethod):
+ def __call__(self, run_args, *args, **kwargs):
+ super(FailBuild, self).__call__(run_args, *args, **kwargs)
+ if run_args[0] == "docker":
+ raise subprocess.CalledProcessError(1, run_args)
+
+ self.useFixture(FakeLogger())
+ args = [
+ "build-docker",
+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+ "--branch", "lp:foo", "test-image",
+ ]
+ build_docker = parse_args(args=args).operation
+ build_docker.backend.build_path = self.useFixture(TempDir()).path
+ build_docker.backend.run = FailBuild()
+ self.assertEqual(RETCODE_FAILURE_BUILD, build_docker.run())
=== added file 'lpbuildd/tests/test_docker.py'
--- lpbuildd/tests/test_docker.py 1970-01-01 00:00:00 +0000
+++ lpbuildd/tests/test_docker.py 2019-07-05 15:18:01 +0000
@@ -0,0 +1,197 @@
+# Copyright 2019 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+__metaclass__ = type
+
+import os
+
+from fixtures import (
+ EnvironmentVariable,
+ TempDir,
+ )
+from testtools import TestCase
+from testtools.deferredruntest import AsynchronousDeferredRunTest
+from twisted.internet import defer
+
+from lpbuildd.docker import (
+ DockerBuildManager,
+ DockerBuildState,
+ )
+from lpbuildd.tests.fakebuilder import FakeBuilder
+from lpbuildd.tests.matchers import HasWaitingFiles
+
+
+class MockBuildManager(DockerBuildManager):
+ def __init__(self, *args, **kwargs):
+ super(MockBuildManager, self).__init__(*args, **kwargs)
+ self.commands = []
+ self.iterators = []
+
+ def runSubProcess(self, path, command, iterate=None, env=None):
+ self.commands.append([path] + command)
+ if iterate is None:
+ iterate = self.iterate
+ self.iterators.append(iterate)
+ return 0
+
+
+class TestDockerBuildManagerIteration(TestCase):
+ """Run DockerBuildManager through its iteration steps."""
+
+ run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=5)
+
+ def setUp(self):
+ super(TestDockerBuildManagerIteration, self).setUp()
+ self.working_dir = self.useFixture(TempDir()).path
+ builder_dir = os.path.join(self.working_dir, "builder")
+ home_dir = os.path.join(self.working_dir, "home")
+ for dir in (builder_dir, home_dir):
+ os.mkdir(dir)
+ self.useFixture(EnvironmentVariable("HOME", home_dir))
+ self.builder = FakeBuilder(builder_dir)
+ self.buildid = "123"
+ self.buildmanager = MockBuildManager(self.builder, self.buildid)
+ self.buildmanager._cachepath = self.builder._cachepath
+
+ def getState(self):
+ """Retrieve build manager's state."""
+ return self.buildmanager._state
+
+ @defer.inlineCallbacks
+ def startBuild(self, args=None, options=None):
+ # The build manager's iterate() kicks off the consecutive states
+ # after INIT.
+ extra_args = {
+ "series": "xenial",
+ "arch_tag": "i386",
+ "name": "test-image",
+ }
+ if args is not None:
+ extra_args.update(args)
+ original_backend_name = self.buildmanager.backend_name
+ self.buildmanager.backend_name = "fake"
+ self.buildmanager.initiate({}, "chroot.tar.gz", extra_args)
+ self.buildmanager.backend_name = original_backend_name
+
+ # Skip states that are done in DebianBuildManager to the state
+ # directly before BUILD_DOCKER.
+ self.buildmanager._state = DockerBuildState.UPDATE
+
+ # BUILD_DOCKER: Run the builder's payload to build the snap package.
+ yield self.buildmanager.iterate(0)
+ self.assertEqual(DockerBuildState.BUILD_DOCKER, self.getState())
+ expected_command = [
+ "sharepath/bin/in-target", "in-target", "build-docker",
+ "--backend=lxd", "--series=xenial", "--arch=i386", self.buildid,
+ ]
+ if options is not None:
+ expected_command.extend(options)
+ expected_command.append("test-image")
+ self.assertEqual(expected_command, self.buildmanager.commands[-1])
+ self.assertEqual(
+ self.buildmanager.iterate, self.buildmanager.iterators[-1])
+ self.assertFalse(self.builder.wasCalled("chrootFail"))
+
+ @defer.inlineCallbacks
+ def test_iterate(self):
+ # The build manager iterates a normal build from start to finish.
+ args = {
+ "git_repository": "https://git.launchpad.dev/~example/+git/snap";,
+ "git_path": "master",
+ }
+ expected_options = [
+ "--git-repository", "https://git.launchpad.dev/~example/+git/snap";,
+ "--git-path", "master",
+ ]
+ yield self.startBuild(args, expected_options)
+
+ log_path = os.path.join(self.buildmanager._cachepath, "buildlog")
+ with open(log_path, "w") as log:
+ log.write("I am a build log.")
+
+ self.buildmanager.backend.add_file("/build/manifest.json", b"[]")
+
+ # After building the package, reap processes.
+ yield self.buildmanager.iterate(0)
+ expected_command = [
+ "sharepath/bin/in-target", "in-target", "scan-for-processes",
+ "--backend=lxd", "--series=xenial", "--arch=i386", self.buildid,
+ ]
+ self.assertEqual(DockerBuildState.BUILD_DOCKER, self.getState())
+ self.assertEqual(expected_command, self.buildmanager.commands[-1])
+ self.assertNotEqual(
+ self.buildmanager.iterate, self.buildmanager.iterators[-1])
+ self.assertFalse(self.builder.wasCalled("buildFail"))
+ self.assertThat(self.builder, HasWaitingFiles.byEquality({
+ "manifest.json": b"[]",
+ }))
+
+ # Control returns to the DebianBuildManager in the UMOUNT state.
+ self.buildmanager.iterateReap(self.getState(), 0)
+ expected_command = [
+ "sharepath/bin/in-target", "in-target", "umount-chroot",
+ "--backend=lxd", "--series=xenial", "--arch=i386", self.buildid,
+ ]
+ self.assertEqual(DockerBuildState.UMOUNT, self.getState())
+ self.assertEqual(expected_command, self.buildmanager.commands[-1])
+ self.assertEqual(
+ self.buildmanager.iterate, self.buildmanager.iterators[-1])
+ self.assertFalse(self.builder.wasCalled("buildFail"))
+
+ @defer.inlineCallbacks
+ def test_iterate_with_file(self):
+ # The build manager iterates a build that specifies a non-default
+ # Dockerfile location from start to finish.
+ args = {
+ "git_repository": "https://git.launchpad.dev/~example/+git/snap";,
+ "git_path": "master",
+ "file": "build-aux/Dockerfile",
+ }
+ expected_options = [
+ "--git-repository", "https://git.launchpad.dev/~example/+git/snap";,
+ "--git-path", "master",
+ "--file", "build-aux/Dockerfile",
+ ]
+ yield self.startBuild(args, expected_options)
+
+ log_path = os.path.join(self.buildmanager._cachepath, "buildlog")
+ with open(log_path, "w") as log:
+ log.write("I am a build log.")
+
+ self.buildmanager.backend.add_file("/build/manifest.json", b"[]")
+
+ # After building the package, reap processes.
+ yield self.buildmanager.iterate(0)
+ expected_command = [
+ "sharepath/bin/in-target", "in-target", "scan-for-processes",
+ "--backend=lxd", "--series=xenial", "--arch=i386", self.buildid,
+ ]
+ self.assertEqual(DockerBuildState.BUILD_DOCKER, self.getState())
+ self.assertEqual(expected_command, self.buildmanager.commands[-1])
+ self.assertNotEqual(
+ self.buildmanager.iterate, self.buildmanager.iterators[-1])
+ self.assertFalse(self.builder.wasCalled("buildFail"))
+ self.assertThat(self.builder, HasWaitingFiles.byEquality({
+ "manifest.json": b"[]",
+ }))
+
+ # Control returns to the DebianBuildManager in the UMOUNT state.
+ self.buildmanager.iterateReap(self.getState(), 0)
+ expected_command = [
+ "sharepath/bin/in-target", "in-target", "umount-chroot",
+ "--backend=lxd", "--series=xenial", "--arch=i386", self.buildid,
+ ]
+ self.assertEqual(DockerBuildState.UMOUNT, self.getState())
+ self.assertEqual(expected_command, self.buildmanager.commands[-1])
+ self.assertEqual(
+ self.buildmanager.iterate, self.buildmanager.iterators[-1])
+ self.assertFalse(self.builder.wasCalled("buildFail"))
+
+ @defer.inlineCallbacks
+ def test_iterate_snap_store_proxy(self):
+ # The build manager can be told to use a snap store proxy.
+ self.builder._config.set(
+ "proxy", "snapstore", "http://snap-store-proxy.example/";)
+ expected_options = [
+ "--snap-store-proxy-url", "http://snap-store-proxy.example/";]
+ yield self.startBuild(options=expected_options)
