launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #21799
[Merge] lp:~cjwatson/launchpad-buildd/lxd-backend into lp:launchpad-buildd
Colin Watson has proposed merging lp:~cjwatson/launchpad-buildd/lxd-backend into lp:launchpad-buildd with lp:~cjwatson/launchpad-buildd/build-snap-operation as a prerequisite.
Commit message:
Add a LXD backend.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad-buildd/lxd-backend/+merge/328666
This would normally also require adding an extra choice to --backend in lpbuildd.target.operation:Operation.make_parser, but I already left lxd in that list of choices by mistake in an earlier branch in this series.
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad-buildd/lxd-backend into lp:launchpad-buildd.
=== modified file 'debian/changelog'
--- debian/changelog 2017-08-07 14:35:18 +0000
+++ debian/changelog 2017-08-07 14:35:18 +0000
@@ -22,6 +22,7 @@
* Rewrite scan-for-processes in Python, allowing it to have unit tests.
* Convert buildlivefs to the new Operation framework and add unit tests.
* Convert buildsnap to the new Operation framework and add unit tests.
+ * Add a LXD backend.
-- Colin Watson <cjwatson@xxxxxxxxxx> Tue, 25 Jul 2017 23:07:58 +0100
=== modified file 'debian/control'
--- debian/control 2017-08-07 14:35:18 +0000
+++ debian/control 2017-08-07 14:35:18 +0000
@@ -21,7 +21,7 @@
Package: python-lpbuildd
Section: python
Architecture: all
-Depends: python, python-twisted-core, python-twisted-web, python-zope.interface, python-apt, python-debian (>= 0.1.23), apt-utils, ${misc:Depends}
+Depends: python, python-twisted-core, python-twisted-web, python-zope.interface, python-apt, python-debian (>= 0.1.23), python-netaddr, apt-utils, ${misc:Depends}
Breaks: launchpad-buildd (<< 88)
Replaces: launchpad-buildd (<< 88)
Description: Python libraries for a Launchpad buildd slave
=== modified file 'lpbuildd/target/backend.py'
--- lpbuildd/target/backend.py 2017-08-07 14:35:18 +0000
+++ lpbuildd/target/backend.py 2017-08-07 14:35:18 +0000
@@ -27,6 +27,9 @@
if name == "chroot":
from lpbuildd.target.chroot import Chroot
backend_factory = Chroot
+ elif name == "lxd":
+ from lpbuildd.target.lxd import LXD
+ backend_factory = LXD
elif name == "fake":
# Only for use in tests.
from lpbuildd.tests.fakeslave import FakeBackend
=== added file 'lpbuildd/target/lxd.py'
--- lpbuildd/target/lxd.py 1970-01-01 00:00:00 +0000
+++ lpbuildd/target/lxd.py 2017-08-07 14:35:18 +0000
@@ -0,0 +1,384 @@
+# Copyright 2017 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from __future__ import print_function
+
+__metaclass__ = type
+
+import io
+import json
+import os
+import shutil
+import stat
+import subprocess
+import tarfile
+import tempfile
+from textwrap import dedent
+import time
+
+import netaddr
+
+from lpbuildd.target.backend import (
+ Backend,
+ BackendException,
+ )
+from lpbuildd.util import (
+ set_personality,
+ shell_escape,
+ )
+
+
+class LXD(Backend):
+
+ # Architecture mapping
+ arches = {
+ "amd64": "x86_64",
+ "arm64": "aarch64",
+ "armhf": "armv7l",
+ "i386": "i686",
+ "powerpc": "ppc",
+ "ppc64el": "ppc64le",
+ "s390x": "s390x",
+ }
+
+ profile_name = "lpbuildd"
+ bridge_name = "lpbr0"
+ # XXX cjwatson 2017-08-07: Hardcoded for now to be in a range reserved
+ # for employee private networks in
+ # https://wiki.canonical.com/InformationInfrastructure/IS/Network, so it
+ # won't collide with any production networks. This should be
+ # configurable.
+ ipv4_network = netaddr.IPNetwork("10.10.10.1/24")
+ run_dir = "/run/launchpad-buildd"
+
+ @property
+ def lxc_arch(self):
+ return self.arches[self.arch]
+
+ @property
+ def alias(self):
+ return "lp-%s-%s" % (self.series, self.arch)
+
+ @property
+ def name(self):
+ return self.alias
+
+ def profile_exists(self):
+ with open("/dev/null", "w") as devnull:
+ return subprocess.call(
+ ["sudo", "lxc", "profile", "show", self.profile_name],
+ stdout=devnull, stderr=devnull) == 0
+
+ def image_exists(self):
+ with open("/dev/null", "w") as devnull:
+ return subprocess.call(
+ ["sudo", "lxc", "image", "info", self.alias],
+ stdout=devnull, stderr=devnull) == 0
+
+ def container_exists(self):
+ with open("/dev/null", "w") as devnull:
+ return subprocess.call(
+ ["sudo", "lxc", "info", self.name],
+ stdout=devnull, stderr=devnull) == 0
+
+ def is_running(self):
+ try:
+ with open("/dev/null", "w") as devnull:
+ output = subprocess.check_output(
+ ["sudo", "lxc", "info", self.name], stderr=devnull)
+ for line in output.splitlines():
+ if line.strip() == "Status: Running":
+ return True
+ else:
+ return False
+ except Exception:
+ return False
+
+ def _convert(self, source_tarball, target_tarball):
+ creation_time = source_tarball.getmember("chroot-autobuild").mtime
+ metadata = {
+ "architecture": self.lxc_arch,
+ "creation_date": creation_time,
+ "properties": {
+ "os": "Ubuntu",
+ "series": self.series,
+ "architecture": self.arch,
+ "description": "Launchpad chroot for Ubuntu %s (%s)" % (
+ self.series, self.arch),
+ },
+ }
+ # Encoding this as JSON is good enough, and saves pulling in a YAML
+ # library dependency.
+ metadata_yaml = json.dumps(
+ metadata, sort_keys=True, indent=4, separators=(",", ": "),
+ ensure_ascii=False).encode("UTF-8") + b"\n"
+ metadata_file = tarfile.TarInfo()
+ metadata_file.size = len(metadata_yaml)
+ metadata_file.name = "metadata.yaml"
+ target_tarball.addfile(metadata_file, io.BytesIO(metadata_yaml))
+
+ copy_from_host = {"/etc/hosts", "/etc/hostname", "/etc/resolv.conf"}
+
+ for entry in source_tarball:
+ fileptr = None
+ try:
+ orig_name = entry.name.split("chroot-autobuild", 1)[-1]
+ entry.name = "rootfs" + orig_name
+
+ if entry.isfile():
+ if orig_name in copy_from_host:
+ target_tarball.add(
+ os.path.realpath(orig_name), arcname=entry.name)
+ continue
+ elif orig_name == "/usr/local/sbin/policy-rc.d":
+ new_bytes = dedent("""\
+ #! /bin/sh
+ while :; do
+ case "$1" in
+ -*) shift ;;
+ snapd) exit 0 ;;
+ *)
+ echo "Not running services in chroot."
+ exit 101
+ ;;
+ esac
+ done
+ """).encode("UTF-8")
+ entry.size = len(new_bytes)
+ fileptr = io.BytesIO(new_bytes)
+ else:
+ try:
+ fileptr = source_tarball.extractfile(entry.name)
+ except KeyError:
+ pass
+ elif entry.islnk():
+ # Update hardlinks to point to the right target
+ entry.linkname = (
+ "rootfs" +
+ entry.linkname.split("chroot-autobuild", 1)[-1])
+
+ target_tarball.addfile(entry, fileobj=fileptr)
+ finally:
+ if fileptr is not None:
+ fileptr.close()
+
+ def create(self, tarball_path):
+ """See `Backend`."""
+ if self.image_exists():
+ self.remove_image()
+
+ tempdir = tempfile.mkdtemp()
+ try:
+ target_path = os.path.join(tempdir, "lxd.tar.gz")
+ with tarfile.open(tarball_path, "r") as source_tarball:
+ with tarfile.open(target_path, "w:gz") as target_tarball:
+ self._convert(source_tarball, target_tarball)
+
+ with open("/dev/null", "w") as devnull:
+ subprocess.check_call(
+ ["sudo", "lxc", "image", "import", target_path,
+ "--alias", self.alias], stdout=devnull)
+ finally:
+ shutil.rmtree(tempdir)
+
+ @property
+ def sys_dir(self):
+ return os.path.join("/sys/class/net", self.bridge_name)
+
+ @property
+ def dnsmasq_pid_file(self):
+ return os.path.join(self.run_dir, "dnsmasq.pid")
+
+ def iptables(self, args, check=True):
+ call = subprocess.check_call if check else subprocess.call
+ call(
+ ["sudo", "iptables", "-w"] + args +
+ ["-m", "comment", "--comment", "managed by launchpad-buildd"])
+
+ def start_bridge(self):
+ if not os.path.isdir(self.run_dir):
+ os.makedirs(self.run_dir)
+ subprocess.check_call(
+ ["sudo", "ip", "link", "add", "dev", self.bridge_name,
+ "type", "bridge"])
+ subprocess.check_call(
+ ["sudo", "ip", "addr", "add", str(self.ipv4_network),
+ "dev", self.bridge_name])
+ subprocess.check_call(
+ ["sudo", "ip", "link", "set", "dev", self.bridge_name, "up"])
+ subprocess.check_call(
+ ["sudo", "sh", "-c", "echo 1 >/proc/sys/net/ipv4/ip_forward"])
+ self.iptables(
+ ["-t", "nat", "-A", "POSTROUTING",
+ "-s", str(self.ipv4_network), "!", "-d", str(self.ipv4_network),
+ "-j", "MASQUERADE"])
+ for protocol in ("udp", "tcp"):
+ self.iptables(
+ ["-I", "INPUT", "-i", self.bridge_name,
+ "-p", protocol, "--dport", "53", "-j", "ACCEPT"])
+ self.iptables(
+ ["-I", "FORWARD", "-i", self.bridge_name, "-j", "ACCEPT"])
+ self.iptables(
+ ["-I", "FORWARD", "-o", self.bridge_name, "-j", "ACCEPT"])
+ subprocess.check_call(
+ ["sudo", "/usr/sbin/dnsmasq", "-s", "lpbuildd", "-S", "/lpbuildd/",
+ "-u", "buildd", "--strict-order", "--bind-interfaces",
+ "--pid-file=%s" % self.dnsmasq_pid_file,
+ "--except-interface=lo", "--interface=%s" % self.bridge_name,
+ "--listen-address=%s" % str(self.ipv4_network.ip)])
+
+ def stop_bridge(self):
+ if not os.path.isdir(self.sys_dir):
+ return
+ subprocess.call(
+ ["sudo", "ip", "addr", "flush", "dev", self.bridge_name])
+ subprocess.call(
+ ["sudo", "ip", "link", "set", "dev", self.bridge_name, "down"])
+ for protocol in ("udp", "tcp"):
+ self.iptables(
+ ["-D", "INPUT", "-i", self.bridge_name,
+ "-p", protocol, "--dport", "53", "-j", "ACCEPT"], check=False)
+ self.iptables(
+ ["-D", "FORWARD", "-i", self.bridge_name, "-j", "ACCEPT"],
+ check=False)
+ self.iptables(
+ ["-D", "FORWARD", "-o", self.bridge_name, "-j", "ACCEPT"],
+ check=False)
+ self.iptables(
+ ["-t", "nat", "-D", "POSTROUTING",
+ "-s", str(self.ipv4_network), "!", "-d", str(self.ipv4_network),
+ "-j", "MASQUERADE"], check=False)
+ if os.path.exists(self.dnsmasq_pid_file):
+ with open(self.dnsmasq_pid_file) as f:
+ try:
+ dnsmasq_pid = int(f.read())
+ except Exception:
+ pass
+ else:
+ # dnsmasq is supposed to drop privileges, but kill it as
+ # root just in case it fails to do so for some reason.
+ subprocess.call(["sudo", "kill", "-9", str(dnsmasq_pid)])
+ os.unlink(self.dnsmasq_pid_file)
+ subprocess.call(["sudo", "ip", "link", "delete", self.bridge_name])
+
+ def start(self):
+ """See `Backend`."""
+ self.stop()
+
+ for addr in self.ipv4_network:
+ if addr not in (
+ self.ipv4_network.network, self.ipv4_network.ip,
+ self.ipv4_network.broadcast):
+ ipv4_address = netaddr.IPNetwork(
+ (int(addr), self.ipv4_network.prefixlen))
+ break
+ else:
+ raise BackendException(
+ "%s has no usable IP addresses" % self.ipv4_network)
+
+ if self.profile_exists():
+ with open("/dev/null", "w") as devnull:
+ subprocess.check_call(
+ ["sudo", "lxc", "profile", "delete", self.profile_name],
+ stdout=devnull)
+ subprocess.check_call(
+ ["sudo", "lxc", "profile", "copy", "default", self.profile_name])
+ subprocess.check_call(
+ ["sudo", "lxc", "profile", "device", "set", self.profile_name,
+ "eth0", "parent", self.bridge_name])
+
+ def set_key(key, value):
+ subprocess.check_call(
+ ["sudo", "lxc", "profile", "set", self.profile_name,
+ key, value])
+
+ set_key("security.privileged", "true")
+ set_key("raw.lxc", dedent("""\
+ lxc.aa_profile=unconfined
+ lxc.cgroup.devices.deny=
+ lxc.cgroup.devices.allow=
+ lxc.network.0.ipv4={ipv4_address}
+ lxc.network.0.ipv4.gateway={ipv4_gateway}
+ """.format(
+ ipv4_address=ipv4_address, ipv4_gateway=self.ipv4_network.ip)))
+
+ self.start_bridge()
+
+ subprocess.check_call(
+ ["sudo", "lxc", "init", "--ephemeral", "-p", self.profile_name,
+ self.alias, self.name])
+
+ for path in ("/etc/hosts", "/etc/hostname", "/etc/resolv.conf"):
+ self.copy_in(path, path)
+
+ # Start the container
+ with open("/dev/null", "w") as devnull:
+ subprocess.check_call(
+ ["sudo", "lxc", "start", self.name], stdout=devnull)
+
+ # Wait for container to start
+ timeout = 60
+ now = time.time()
+ while time.time() < now + timeout:
+ if self.is_running():
+ return
+ time.sleep(5)
+ if not self.is_running():
+ raise BackendException(
+ "Container failed to start within %d seconds" % timeout)
+
+ def run(self, args, env=None, input_text=None, get_output=False,
+ echo=False, **kwargs):
+ """See `Backend`."""
+ if env:
+ args = ["env"] + [
+ "%s=%s" % (key, shell_escape(value))
+ for key, value in env.items()] + args
+ if self.arch is not None:
+ args = set_personality(args, self.arch, series=self.series)
+ if echo:
+ print("Running in container: %s" % ' '.join(
+ shell_escape(arg) for arg in args))
+ cmd = ["sudo", "lxc", "exec", self.name, "--"] + args
+ if input_text is None and not get_output:
+ subprocess.check_call(cmd, **kwargs)
+ else:
+ if get_output:
+ kwargs["stdout"] = subprocess.PIPE
+ proc = subprocess.Popen(
+ cmd, stdin=subprocess.PIPE, universal_newlines=True, **kwargs)
+ output, _ = proc.communicate(input_text)
+ if proc.returncode:
+ raise subprocess.CalledProcessError(proc.returncode, cmd)
+ if get_output:
+ return output
+
+ def copy_in(self, source_path, target_path):
+ """See `Backend`."""
+ mode = stat.S_IMODE(os.stat(source_path).st_mode)
+ subprocess.check_call(
+ ["sudo", "lxc", "file", "push",
+ "--uid=0", "--gid=0", "--mode=%o" % mode,
+ source_path, self.name + target_path])
+
+ def copy_out(self, source_path, target_path):
+ subprocess.check_call(
+ ["sudo", "lxc", "file", "pull",
+ self.name + source_path, target_path])
+
+ def stop(self):
+ """See `Backend`."""
+ if self.is_running():
+ subprocess.check_call(["sudo", "lxc", "stop", self.name])
+ if self.container_exists():
+ subprocess.check_call(["sudo", "lxc", "delete", self.name])
+ self.stop_bridge()
+
+ def remove_image(self):
+ subprocess.check_call(["sudo", "lxc", "image", "delete", self.alias])
+
+ def remove(self):
+ """See `Backend`."""
+ if self.image_exists():
+ self.remove_image()
+ super(LXD, self).remove()
=== added file 'lpbuildd/target/tests/test_lxd.py'
--- lpbuildd/target/tests/test_lxd.py 1970-01-01 00:00:00 +0000
+++ lpbuildd/target/tests/test_lxd.py 2017-08-07 14:35:18 +0000
@@ -0,0 +1,467 @@
+# Copyright 2017 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+__metaclass__ = type
+
+import io
+import json
+import os.path
+import tarfile
+from textwrap import dedent
+
+from fixtures import (
+ EnvironmentVariable,
+ MonkeyPatch,
+ TempDir,
+ )
+from systemfixtures import (
+ FakeFilesystem,
+ FakeProcesses,
+ )
+from testtools import TestCase
+from testtools.matchers import (
+ DirContains,
+ EndsWith,
+ Equals,
+ FileContains,
+ HasPermissions,
+ MatchesDict,
+ MatchesListwise,
+ )
+
+from lpbuildd.target.lxd import LXD
+
+
+class TestLXD(TestCase):
+
+ def make_chroot_tarball(self, output_path):
+ source = self.useFixture(TempDir()).path
+ hello = os.path.join(source, "bin", "hello")
+ os.mkdir(os.path.dirname(hello))
+ with open(hello, "w") as f:
+ f.write("hello\n")
+ os.fchmod(f.fileno(), 0o755)
+ os.mkdir(os.path.join(source, "etc"))
+ for name in ("hosts", "hostname", "resolv.conf"):
+ with open(os.path.join(source, "etc", name), "w") as f:
+ f.write("%s\n" % name)
+ policy_rc_d = os.path.join(
+ source, "usr", "local", "sbin", "policy-rc.d")
+ os.makedirs(os.path.dirname(policy_rc_d))
+ with open(policy_rc_d, "w") as f:
+ f.write("original policy-rc.d\n")
+ os.fchmod(f.fileno(), 0o755)
+ with tarfile.open(output_path, "w:bz2") as tar:
+ tar.add(source, arcname="chroot-autobuild")
+
+ def make_fake_etc(self):
+ fs_fixture = self.useFixture(FakeFilesystem())
+ fs_fixture.add("/etc")
+ os.mkdir("/etc")
+ for name in ("hosts", "hostname", "resolv.conf"):
+ with open(os.path.join("/etc", name), "w") as f:
+ f.write("host %s\n" % name)
+ # systemfixtures doesn't patch this, but arguably should.
+ self.useFixture(MonkeyPatch("tarfile.bltn_open", open))
+
+ def test_convert(self):
+ tmp = self.useFixture(TempDir()).path
+ source_tarball_path = os.path.join(tmp, "source.tar.bz2")
+ target_tarball_path = os.path.join(tmp, "target.tar.gz")
+ self.make_chroot_tarball(source_tarball_path)
+ self.make_fake_etc()
+ with tarfile.open(source_tarball_path, "r") as source_tarball:
+ creation_time = source_tarball.getmember("chroot-autobuild").mtime
+ with tarfile.open(target_tarball_path, "w:gz") as target_tarball:
+ LXD("1", "xenial", "amd64")._convert(
+ source_tarball, target_tarball)
+
+ target = os.path.join(tmp, "target")
+ with tarfile.open(target_tarball_path, "r") as target_tarball:
+ target_tarball.extractall(path=target)
+ self.assertThat(target, DirContains(["metadata.yaml", "rootfs"]))
+ with open(os.path.join(target, "metadata.yaml")) as metadata_file:
+ metadata = json.load(metadata_file)
+ self.assertThat(metadata, MatchesDict({
+ "architecture": Equals("x86_64"),
+ "creation_date": Equals(creation_time),
+ "properties": MatchesDict({
+ "os": Equals("Ubuntu"),
+ "series": Equals("xenial"),
+ "architecture": Equals("amd64"),
+ "description": Equals(
+ "Launchpad chroot for Ubuntu xenial (amd64)"),
+ }),
+ }))
+ rootfs = os.path.join(target, "rootfs")
+ self.assertThat(rootfs, DirContains(["bin", "etc", "usr"]))
+ self.assertThat(os.path.join(rootfs, "bin"), DirContains(["hello"]))
+ hello = os.path.join(rootfs, "bin", "hello")
+ self.assertThat(hello, FileContains("hello\n"))
+ self.assertThat(hello, HasPermissions("0755"))
+ self.assertThat(
+ os.path.join(rootfs, "etc"),
+ DirContains(["hosts", "hostname", "resolv.conf"]))
+ for name in ("hosts", "hostname", "resolv.conf"):
+ self.assertThat(
+ os.path.join(rootfs, "etc", name),
+ FileContains("host %s\n" % name))
+ policy_rc_d = os.path.join(
+ rootfs, "usr", "local", "sbin", "policy-rc.d")
+ self.assertThat(
+ policy_rc_d,
+ FileContains(dedent("""\
+ #! /bin/sh
+ while :; do
+ case "$1" in
+ -*) shift ;;
+ snapd) exit 0 ;;
+ *)
+ echo "Not running services in chroot."
+ exit 101
+ ;;
+ esac
+ done
+ """)))
+ self.assertThat(policy_rc_d, HasPermissions("0755"))
+
+ def test_create(self):
+ tmp = self.useFixture(TempDir()).path
+ source_tarball_path = os.path.join(tmp, "source.tar.bz2")
+ self.make_chroot_tarball(source_tarball_path)
+ self.make_fake_etc()
+ processes_fixture = self.useFixture(FakeProcesses())
+ processes_fixture.add(
+ lambda proc_args: {
+ "returncode": 1 if "info" in proc_args["args"] else 0,
+ },
+ name="sudo")
+ LXD("1", "xenial", "amd64").create(source_tarball_path)
+
+ self.assertThat(
+ [proc._args["args"] for proc in processes_fixture.procs],
+ MatchesListwise([
+ Equals(["sudo", "lxc", "image", "info", "lp-xenial-amd64"]),
+ MatchesListwise([
+ Equals("sudo"), Equals("lxc"), Equals("image"),
+ Equals("import"), EndsWith("/lxd.tar.gz"),
+ Equals("--alias"), Equals("lp-xenial-amd64"),
+ ]),
+ ]))
+
+ def test_start(self):
+ class SudoLXC:
+ def __init__(self):
+ self.created = False
+ self.started = False
+
+ def __call__(self, proc_info):
+ ret = {}
+ if proc_info["args"][:4] == ["sudo", "lxc", "profile", "show"]:
+ ret["returncode"] = 1
+ elif proc_info["args"][:3] == ["sudo", "lxc", "init"]:
+ self.created = True
+ elif proc_info["args"][:3] == ["sudo", "lxc", "start"]:
+ self.started = True
+ elif proc_info["args"][:3] == ["sudo", "lxc", "info"]:
+ if not self.created:
+ ret["returncode"] = 1
+ else:
+ status = "Running" if self.started else "Stopped"
+ ret["stdout"] = io.BytesIO(
+ ("Status: %s\n" % status).encode("UTF-8"))
+ return ret
+
+ fs_fixture = self.useFixture(FakeFilesystem())
+ fs_fixture.add("/sys")
+ fs_fixture.add("/run")
+ os.makedirs("/run/launchpad-buildd")
+ fs_fixture.add("/etc")
+ os.mkdir("/etc")
+ for name in ("hosts", "hostname", "resolv.conf"):
+ with open(os.path.join("/etc", name), "w") as f:
+ f.write("host %s\n" % name)
+ processes_fixture = self.useFixture(FakeProcesses())
+ processes_fixture.add(SudoLXC(), name="sudo")
+ LXD("1", "xenial", "amd64").start()
+
+ lxc = ["sudo", "lxc"]
+ raw_lxc = dedent("""\
+ lxc.aa_profile=unconfined
+ lxc.cgroup.devices.deny=
+ lxc.cgroup.devices.allow=
+ lxc.network.0.ipv4=10.10.10.2/24
+ lxc.network.0.ipv4.gateway=10.10.10.1
+ """)
+ ip = ["sudo", "ip"]
+ iptables = ["sudo", "iptables", "-w"]
+ iptables_comment = [
+ "-m", "comment", "--comment", "managed by launchpad-buildd"]
+ self.assertThat(
+ [proc._args["args"] for proc in processes_fixture.procs],
+ MatchesListwise([
+ Equals(lxc + ["info", "lp-xenial-amd64"]),
+ Equals(lxc + ["info", "lp-xenial-amd64"]),
+ Equals(lxc + ["profile", "show", "lpbuildd"]),
+ Equals(lxc + ["profile", "copy", "default", "lpbuildd"]),
+ Equals(lxc + ["profile", "device", "set", "lpbuildd", "eth0",
+ "parent", "lpbr0"]),
+ Equals(lxc + ["profile", "set", "lpbuildd",
+ "security.privileged", "true"]),
+ Equals(lxc + ["profile", "set", "lpbuildd",
+ "raw.lxc", raw_lxc]),
+ Equals(ip + ["link", "add", "dev", "lpbr0", "type", "bridge"]),
+ Equals(ip + ["addr", "add", "10.10.10.1/24", "dev", "lpbr0"]),
+ Equals(ip + ["link", "set", "dev", "lpbr0", "up"]),
+ Equals(
+ ["sudo", "sh", "-c",
+ "echo 1 >/proc/sys/net/ipv4/ip_forward"]),
+ Equals(
+ iptables +
+ ["-t", "nat", "-A", "POSTROUTING",
+ "-s", "10.10.10.1/24", "!", "-d", "10.10.10.1/24",
+ "-j", "MASQUERADE"] +
+ iptables_comment),
+ Equals(
+ iptables +
+ ["-I", "INPUT", "-i", "lpbr0",
+ "-p", "udp", "--dport", "53", "-j", "ACCEPT"] +
+ iptables_comment),
+ Equals(
+ iptables +
+ ["-I", "INPUT", "-i", "lpbr0",
+ "-p", "tcp", "--dport", "53", "-j", "ACCEPT"] +
+ iptables_comment),
+ Equals(
+ iptables +
+ ["-I", "FORWARD", "-i", "lpbr0", "-j", "ACCEPT"] +
+ iptables_comment),
+ Equals(
+ iptables +
+ ["-I", "FORWARD", "-o", "lpbr0", "-j", "ACCEPT"] +
+ iptables_comment),
+ Equals(
+ ["sudo", "/usr/sbin/dnsmasq", "-s", "lpbuildd",
+ "-S", "/lpbuildd/", "-u", "buildd", "--strict-order",
+ "--bind-interfaces",
+ "--pid-file=/run/launchpad-buildd/dnsmasq.pid",
+ "--except-interface=lo", "--interface=lpbr0",
+ "--listen-address=10.10.10.1"]),
+ Equals(lxc + ["init", "--ephemeral", "-p", "lpbuildd",
+ "lp-xenial-amd64", "lp-xenial-amd64"]),
+ Equals(lxc + ["file", "push",
+ "--uid=0", "--gid=0", "--mode=644",
+ "/etc/hosts", "lp-xenial-amd64/etc/hosts"]),
+ Equals(lxc + ["file", "push",
+ "--uid=0", "--gid=0", "--mode=644",
+ "/etc/hostname",
+ "lp-xenial-amd64/etc/hostname"]),
+ Equals(lxc + ["file", "push",
+ "--uid=0", "--gid=0", "--mode=644",
+ "/etc/resolv.conf",
+ "lp-xenial-amd64/etc/resolv.conf"]),
+ Equals(lxc + ["start", "lp-xenial-amd64"]),
+ Equals(lxc + ["info", "lp-xenial-amd64"]),
+ ]))
+
+ def test_run(self):
+ processes_fixture = self.useFixture(FakeProcesses())
+ processes_fixture.add(lambda _: {}, name="sudo")
+ LXD("1", "xenial", "amd64").run(
+ ["apt-get", "update"], env={"LANG": "C"})
+
+ expected_args = [
+ ["sudo", "lxc", "exec", "lp-xenial-amd64", "--",
+ "linux64", "env", "LANG=C", "apt-get", "update"],
+ ]
+ self.assertEqual(
+ expected_args,
+ [proc._args["args"] for proc in processes_fixture.procs])
+
+ def test_run_get_output(self):
+ processes_fixture = self.useFixture(FakeProcesses())
+ processes_fixture.add(
+ lambda _: {"stdout": io.BytesIO(b"hello\n")}, name="sudo")
+ self.assertEqual(
+ "hello\n",
+ LXD("1", "xenial", "amd64").run(
+ ["echo", "hello"], get_output=True))
+
+ expected_args = [
+ ["sudo", "lxc", "exec", "lp-xenial-amd64", "--",
+ "linux64", "echo", "hello"],
+ ]
+ self.assertEqual(
+ expected_args,
+ [proc._args["args"] for proc in processes_fixture.procs])
+
+ def test_copy_in(self):
+ source_dir = self.useFixture(TempDir()).path
+ processes_fixture = self.useFixture(FakeProcesses())
+ processes_fixture.add(lambda _: {}, name="sudo")
+ source_path = os.path.join(source_dir, "source")
+ with open(source_path, "w"):
+ pass
+ os.chmod(source_path, 0o644)
+ target_path = "/path/to/target"
+ LXD("1", "xenial", "amd64").copy_in(source_path, target_path)
+
+ expected_args = [
+ ["sudo", "lxc", "file", "push", "--uid=0", "--gid=0", "--mode=644",
+ source_path, "lp-xenial-amd64" + target_path],
+ ]
+ self.assertEqual(
+ expected_args,
+ [proc._args["args"] for proc in processes_fixture.procs])
+
+ def test_copy_out(self):
+ processes_fixture = self.useFixture(FakeProcesses())
+ processes_fixture.add(lambda _: {}, name="sudo")
+ LXD("1", "xenial", "amd64").copy_out(
+ "/path/to/source", "/path/to/target")
+
+ expected_args = [
+ ["sudo", "lxc", "file", "pull",
+ "lp-xenial-amd64/path/to/source", "/path/to/target"],
+ ]
+ self.assertEqual(
+ expected_args,
+ [proc._args["args"] for proc in processes_fixture.procs])
+
+ def test_path_exists(self):
+ processes_fixture = self.useFixture(FakeProcesses())
+ test_proc_infos = iter([{}, {"returncode": 1}])
+ processes_fixture.add(lambda _: next(test_proc_infos), name="sudo")
+ self.assertTrue(LXD("1", "xenial", "amd64").path_exists("/present"))
+ self.assertFalse(LXD("1", "xenial", "amd64").path_exists("/absent"))
+
+ expected_args = [
+ ["sudo", "lxc", "exec", "lp-xenial-amd64", "--",
+ "linux64", "test", "-e", path]
+ for path in ("/present", "/absent")
+ ]
+ self.assertEqual(
+ expected_args,
+ [proc._args["args"] for proc in processes_fixture.procs])
+
+ def test_islink(self):
+ processes_fixture = self.useFixture(FakeProcesses())
+ test_proc_infos = iter([{}, {"returncode": 1}])
+ processes_fixture.add(lambda _: next(test_proc_infos), name="sudo")
+ self.assertTrue(LXD("1", "xenial", "amd64").islink("/link"))
+ self.assertFalse(LXD("1", "xenial", "amd64").islink("/file"))
+
+ expected_args = [
+ ["sudo", "lxc", "exec", "lp-xenial-amd64", "--",
+ "linux64", "test", "-h", path]
+ for path in ("/link", "/file")
+ ]
+ self.assertEqual(
+ expected_args,
+ [proc._args["args"] for proc in processes_fixture.procs])
+
+ def test_listdir(self):
+ processes_fixture = self.useFixture(FakeProcesses())
+ processes_fixture.add(
+ lambda _: {"stdout": io.BytesIO(b"foo\0bar\0baz\0")}, name="sudo")
+ self.assertEqual(
+ ["foo", "bar", "baz"],
+ LXD("1", "xenial", "amd64").listdir("/path"))
+
+ expected_args = [
+ ["sudo", "lxc", "exec", "lp-xenial-amd64", "--",
+ "linux64", "find", "/path", "-mindepth", "1", "-maxdepth", "1",
+ "-printf", "%P\\0"],
+ ]
+ self.assertEqual(
+ expected_args,
+ [proc._args["args"] for proc in processes_fixture.procs])
+
+ def test_stop(self):
+ class SudoLXC:
+ def __init__(self):
+ self.stopped = False
+ self.deleted = False
+
+ def __call__(self, proc_info):
+ ret = {}
+ if proc_info["args"][:3] == ["sudo", "lxc", "stop"]:
+ self.stopped = True
+ elif proc_info["args"][:3] == ["sudo", "lxc", "delete"]:
+ self.deleted = True
+ elif proc_info["args"][:3] == ["sudo", "lxc", "info"]:
+ if self.deleted:
+ ret["returncode"] = 1
+ else:
+ status = "Stopped" if self.stopped else "Running"
+ ret["stdout"] = io.BytesIO(
+ ("Status: %s\n" % status).encode("UTF-8"))
+ return ret
+
+ fs_fixture = self.useFixture(FakeFilesystem())
+ fs_fixture.add("/sys")
+ os.makedirs("/sys/class/net/lpbr0")
+ fs_fixture.add("/run")
+ os.makedirs("/run/launchpad-buildd")
+ with open("/run/launchpad-buildd/dnsmasq.pid", "w") as f:
+ f.write("42\n")
+ processes_fixture = self.useFixture(FakeProcesses())
+ processes_fixture.add(SudoLXC(), name="sudo")
+ LXD("1", "xenial", "amd64").stop()
+
+ lxc = ["sudo", "lxc"]
+ ip = ["sudo", "ip"]
+ iptables = ["sudo", "iptables", "-w"]
+ iptables_comment = [
+ "-m", "comment", "--comment", "managed by launchpad-buildd"]
+ self.assertThat(
+ [proc._args["args"] for proc in processes_fixture.procs],
+ MatchesListwise([
+ Equals(lxc + ["info", "lp-xenial-amd64"]),
+ Equals(lxc + ["stop", "lp-xenial-amd64"]),
+ Equals(lxc + ["info", "lp-xenial-amd64"]),
+ Equals(lxc + ["delete", "lp-xenial-amd64"]),
+ Equals(ip + ["addr", "flush", "dev", "lpbr0"]),
+ Equals(ip + ["link", "set", "dev", "lpbr0", "down"]),
+ Equals(
+ iptables +
+ ["-D", "INPUT", "-i", "lpbr0",
+ "-p", "udp", "--dport", "53", "-j", "ACCEPT"] +
+ iptables_comment),
+ Equals(
+ iptables +
+ ["-D", "INPUT", "-i", "lpbr0",
+ "-p", "tcp", "--dport", "53", "-j", "ACCEPT"] +
+ iptables_comment),
+ Equals(
+ iptables +
+ ["-D", "FORWARD", "-i", "lpbr0", "-j", "ACCEPT"] +
+ iptables_comment),
+ Equals(
+ iptables +
+ ["-D", "FORWARD", "-o", "lpbr0", "-j", "ACCEPT"] +
+ iptables_comment),
+ Equals(
+ iptables +
+ ["-t", "nat", "-D", "POSTROUTING",
+ "-s", "10.10.10.1/24", "!", "-d", "10.10.10.1/24",
+ "-j", "MASQUERADE"] +
+ iptables_comment),
+ Equals(["sudo", "kill", "-9", "42"]),
+ Equals(ip + ["link", "delete", "lpbr0"]),
+ ]))
+
+ def test_remove(self):
+ self.useFixture(EnvironmentVariable("HOME", "/expected/home"))
+ processes_fixture = self.useFixture(FakeProcesses())
+ processes_fixture.add(lambda _: {}, name="sudo")
+ LXD("1", "xenial", "amd64").remove()
+
+ lxc = ["sudo", "lxc"]
+ self.assertThat(
+ [proc._args["args"] for proc in processes_fixture.procs],
+ MatchesListwise([
+ Equals(lxc + ["image", "info", "lp-xenial-amd64"]),
+ Equals(lxc + ["image", "delete", "lp-xenial-amd64"]),
+ Equals(["sudo", "rm", "-rf", "/expected/home/build-1"]),
+ ]))
