launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #16162
[Merge] lp:~cjwatson/launchpad-buildd/livefs into lp:launchpad-buildd
Colin Watson has proposed merging lp:~cjwatson/launchpad-buildd/livefs into lp:launchpad-buildd.
Commit message:
Add a new "livefs" build manager, based on livecd-rootfs/BuildLiveCD.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
Related bugs:
Bug #1247461 in launchpad-buildd: "Move live filesystem building into Launchpad"
https://bugs.launchpad.net/launchpad-buildd/+bug/1247461
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad-buildd/livefs/+merge/193682
This is a first cut at the build slave side of livefs building. It's somewhat based on BuildLiveCD from livecd-rootfs, but refactored into launchpad-buildd style and with a good deal of historical cruft stripped out. I've successfully tested this with an ubuntu-core/saucy/i386 build.
livecd-rootfs notionally has eatmydata support, but I don't think I ever got that deployed and on closer inspection I'm not clear that it ever worked in livecd-rootfs (bad syntax of environment variable handling; libeatmydata.so doesn't exist in the chroot). I've omitted this for the moment, and perhaps we can restore it later.
Note that if you're testing this in an LXC container then you need to use "aa-complain /usr/bin/lxc-start", perhaps due to bug 969299 or at any rate something with similar symptoms (debootstrap fails to unpack udev due to something related to dpkg-divert). This won't be a problem for production deployment.
--
https://code.launchpad.net/~cjwatson/launchpad-buildd/livefs/+merge/193682
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad-buildd/livefs into lp:launchpad-buildd.
=== modified file 'Makefile'
--- Makefile 2013-10-03 10:39:23 +0000
+++ Makefile 2013-11-02 20:02:43 +0000
@@ -32,4 +32,5 @@
lpbuildd.tests.test_check_implicit_pointer_functions \
lpbuildd.tests.test_harness \
lpbuildd.tests.test_translationtemplatesbuildmanager \
- lpbuildd.tests.test_sourcepackagerecipe
+ lpbuildd.tests.test_sourcepackagerecipe \
+ lpbuildd.tests.test_livefs
=== modified file 'buildd-slave.tac'
--- buildd-slave.tac 2011-11-09 08:50:17 +0000
+++ buildd-slave.tac 2013-11-02 20:02:43 +0000
@@ -8,6 +8,7 @@
from twisted.application import service, strports
from lpbuildd.slave import XMLRPCBuildDSlave
from lpbuildd.binarypackage import BinaryPackageBuildManager
+from lpbuildd.livefs import LiveFilesystemBuildManager
from lpbuildd.sourcepackagerecipe import (
SourcePackageRecipeBuildManager)
from lpbuildd.translationtemplates import (
@@ -30,6 +31,7 @@
slave.registerBuilder(SourcePackageRecipeBuildManager, "sourcepackagerecipe")
slave.registerBuilder(
TranslationTemplatesBuildManager, 'translation-templates')
+slave.registerBuilder(LiveFilesystemBuildManager, "livefs")
application = service.Application('BuildDSlave')
builddslaveService = service.IServiceCollection(application)
=== added file 'buildlivefs'
--- buildlivefs 1970-01-01 00:00:00 +0000
+++ buildlivefs 2013-11-02 20:02:43 +0000
@@ -0,0 +1,164 @@
+#! /usr/bin/python -u
+# Copyright 2013 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""A script that builds a live file system."""
+
+__metaclass__ = type
+
+from datetime import date
+from optparse import OptionParser
+import os
+import re
+import subprocess
+import sys
+import traceback
+
+
+RETCODE_SUCCESS = 0
+RETCODE_FAILURE_INSTALL = 200
+RETCODE_FAILURE_BUILD = 201
+
+
+def get_build_path(build_id, *extra):
+ """Generate a path within the build directory.
+
+ :param build_id: the build id to use.
+ :param extra: the extra path segments within the build directory.
+ :return: the generated path.
+ """
+ return os.path.join(os.environ["HOME"], "build-" + build_id, *extra)
+
+
+non_meta_re = re.compile(r'^[a-zA-Z0-9+,./:=@_-]+$')
+
+def shell_escape(arg):
+ if non_meta_re.match(arg):
+ return arg
+ else:
+ return "'%s'" % arg.replace("'", "'\\''")
+
+
+class LiveFSBuilder:
+ """Builds a live file system."""
+
+ def __init__(self, options):
+ self.options = options
+ self.chroot_path = get_build_path(
+ self.options.build_id, 'chroot-autobuild')
+
+ def chroot(self, args, echo=False):
+ """Run a command in the chroot.
+
+ :param args: the command and arguments to run.
+ """
+ if echo:
+ print "Running in chroot: %s" % ' '.join(
+ "'%s'" % arg for arg in args)
+ sys.stdout.flush()
+ subprocess.check_call([
+ "/usr/bin/sudo", "/usr/sbin/chroot", self.chroot_path] + args)
+
+ def run_build_command(self, args, env=None, echo=False):
+ """Run a build command in the chroot.
+
+ This is unpleasant because we need to run it in /build under sudo
+ chroot, and there's no way to do this without either a helper
+ program in the chroot or unpleasant quoting. We go for the
+ unpleasant quoting.
+
+ :param args: the command and arguments to run.
+ :param env: dictionary of additional environment variables to set.
+ """
+ args = [shell_escape(arg) for arg in args]
+ if env:
+ args = ["env"] + [
+ "%s=%s" % (key, shell_escape(value))
+ for key, value in env.items()] + args
+ command = "cd /build && %s" % " ".join(args)
+ self.chroot(["/bin/sh", "-c", command], echo=echo)
+
+ def install(self):
+ self.chroot(["apt-get", "-y", "install", "livecd-rootfs"])
+ if self.options.locale is not None:
+ self.chroot([
+ "apt-get", "-y", "--install-recommends", "install",
+ "ubuntu-defaults-builder",
+ ])
+
+ def build(self):
+ if self.options.locale is not None:
+ self.run_build_command([
+ "ubuntu-defaults-image",
+ "--locale", self.options.locale,
+ "--arch", self.options.arch,
+ "--release", self.options.suite,
+ ])
+ else:
+ self.run_build_command(["rm", "-rf", "auto"])
+ self.run_build_command(["mkdir", "-p", "auto"])
+ for lb_script in ("config", "build", "clean"):
+ lb_script_path = os.path.join(
+ "/usr/share/livecd-rootfs/live-build/auto", lb_script)
+ self.run_build_command(["ln", "-s", lb_script_path, "auto/"])
+ self.run_build_command(["lb", "clean", "--purge"])
+
+ base_lb_env = {
+ "PROJECT": self.options.project,
+ "ARCH": self.options.arch,
+ }
+ if self.options.subproject is not None:
+ base_lb_env["SUBPROJECT"] = self.options.subproject
+ if self.options.subarch is not None:
+ base_lb_env["SUBARCH"] = self.options.subarch
+ lb_env = dict(base_lb_env)
+ lb_env["NOW"] = date.today().strftime("%Y%m%d")
+ lb_env["SUITE"] = self.options.suite
+ if self.options.image_format is not None:
+ lb_env["IMAGEFORMAT"] = self.options.image_format
+ if self.options.proposed:
+ lb_env["PROPOSED"] = "1"
+ self.run_build_command(["lb", "config"], env=lb_env)
+ self.run_build_command(["lb", "build"], env=base_lb_env)
+
+
+def main():
+ parser = OptionParser()
+ parser.add_option("--build-id", help="build identifier")
+ parser.add_option(
+ "--arch", metavar="ARCH", help="build for architecture ARCH")
+ parser.add_option(
+ "--subarch", metavar="SUBARCH",
+ help="build for subarchitecture SUBARCH")
+ parser.add_option(
+ "--project", metavar="PROJECT", help="build for project PROJECT")
+ parser.add_option(
+ "--subproject", metavar="SUBPROJECT",
+ help="build for subproject SUBPROJECT")
+ parser.add_option("--suite", metavar="SUITE", help="build for suite SUITE")
+ parser.add_option(
+ "--image-format", metavar="FORMAT", help="produce an image in FORMAT")
+ parser.add_option(
+ "--proposed", default=False, action="store_true",
+ help="enable use of -proposed pocket")
+ parser.add_option(
+ "--locale", metavar="LOCALE",
+ help="use ubuntu-defaults-image to build an image for LOCALE")
+ options, _ = parser.parse_args()
+
+ builder = LiveFSBuilder(options)
+ try:
+ builder.install()
+ except Exception:
+ traceback.print_exc()
+ return RETCODE_FAILURE_INSTALL
+ try:
+ builder.build()
+ except Exception:
+ traceback.print_exc()
+ return RETCODE_FAILURE_BUILD
+ return RETCODE_SUCCESS
+
+
+if __name__ == "__main__":
+ sys.exit(main())
=== modified file 'debian/changelog'
--- debian/changelog 2013-10-18 13:08:29 +0000
+++ debian/changelog 2013-11-02 20:02:43 +0000
@@ -4,6 +4,8 @@
of other environment variables too (including DISPLAY and TERM), in line
with Debian buildds.
* Make the status XML-RPC method a synonym for status_dict.
+ * Add a new "livefs" build manager, based on livecd-rootfs/BuildLiveCD
+ (LP: #1247461).
-- Colin Watson <cjwatson@xxxxxxxxxx> Fri, 18 Oct 2013 12:24:00 +0100
=== modified file 'debian/rules'
--- debian/rules 2013-10-03 10:57:53 +0000
+++ debian/rules 2013-11-02 20:02:43 +0000
@@ -18,7 +18,7 @@
slavebins = unpack-chroot mount-chroot update-debian-chroot sbuild-package \
scan-for-processes umount-chroot remove-build override-sources-list \
- buildrecipe generate-translation-templates slave-prep
+ buildrecipe generate-translation-templates slave-prep buildlivefs
BUILDDUID=65500
BUILDDGID=65500
=== modified file 'debian/upgrade-config'
--- debian/upgrade-config 2013-07-29 17:37:43 +0000
+++ debian/upgrade-config 2013-11-02 20:02:43 +0000
@@ -130,6 +130,18 @@
in_file.close()
out_file.close()
+def upgrade_to_120():
+ print "Upgrading %s to version 120" % conf_file
+ subprocess.call(["mv", conf_file, conf_file+"-prev115~"])
+ in_file = open(conf_file+"-prev120~", "r")
+ out_file = open(conf_file, "w")
+ out_file.write(in_file.read())
+ out_file.write(
+ "\n[livefilesystemmanager]\n"
+ "buildlivefspath = /usr/share/launchpad-buildd/slavebin/buildlivefs\n")
+ in_file.close()
+ out_file.close()
+
if __name__ == "__main__":
old_version = re.sub(r'[~-].*', '', old_version)
if int(old_version) < 12:
@@ -150,4 +162,5 @@
upgrade_to_110()
if int(old_version) < 115:
upgrade_to_115()
-
+ if int(old_version) < 120:
+ upgrade_to_120()
=== added file 'lpbuildd/livefs.py'
--- lpbuildd/livefs.py 1970-01-01 00:00:00 +0000
+++ lpbuildd/livefs.py 2013-11-02 20:02:43 +0000
@@ -0,0 +1,91 @@
+# Copyright 2013 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+__metaclass__ = type
+
+import os
+import shutil
+
+from lpbuildd.debian import (
+ DebianBuildManager,
+ DebianBuildState,
+ get_build_path,
+ )
+
+
+class LiveFilesystemBuildState(DebianBuildState):
+ BUILD_LIVEFS = "BUILD_LIVEFS"
+
+
+class LiveFilesystemBuildManager(DebianBuildManager):
+ """Build a live filesystem."""
+
+ initial_build_state = LiveFilesystemBuildState.BUILD_LIVEFS
+
+ def __init__(self, slave, buildid, **kwargs):
+ DebianBuildManager.__init__(self, slave, buildid, **kwargs)
+ self.build_livefs_path = slave._config.get(
+ "livefilesystemmanager", "buildlivefspath")
+
+ def initiate(self, files, chroot, extra_args):
+ """Initiate a build with a given set of files and chroot."""
+ self.build_path = get_build_path(
+ self.home, self._buildid, "chroot-autobuild", "build")
+ if os.path.isdir(self.build_path):
+ shutil.rmtree(self.build_path)
+
+ self.subarch = extra_args.get("subarch")
+ self.project = extra_args["project"]
+ self.subproject = extra_args.get("subproject")
+ self.suite = extra_args["suite"]
+ self.image_format = extra_args.get("image_format")
+ self.proposed = extra_args.get("proposed", False)
+ self.locale = extra_args.get("locale")
+
+ super(LiveFilesystemBuildManager, self).initiate(
+ files, chroot, extra_args)
+
+ def doRunBuild(self):
+ """Run the process to build the live filesystem."""
+ args = [
+ "buildlivefs",
+ "--build-id", self._buildid,
+ "--arch", self.arch_tag,
+ ]
+ if self.subarch:
+ args.extend(["--subarch", self.subarch])
+ args.extend(["--project", self.project])
+ if self.subproject:
+ args.extend(["--subproject", self.subproject])
+ args.extend(["--suite", self.suite])
+ if self.image_format:
+ args.extend(["--image-format", self.image_format])
+ if self.proposed:
+ args.append("--proposed")
+ if self.locale:
+ args.extend(["--locale", self.locale])
+ self.runSubProcess(self.build_livefs_path, args)
+
+ def iterate_BUILD_LIVEFS(self, retcode):
+ """Finished building the live filesystem."""
+ if retcode == 0:
+ self.gatherResults()
+ print("Returning build status: OK")
+ else:
+ if not self.alreadyfailed:
+ self._slave.buildFail()
+ print("Returning build status: Build failed.")
+ self.alreadyfailed = True
+ self.doReapProcesses(self._state)
+
+ def iterateReap_BUILD_LIVEFS(self, retcode):
+ """Finished reaping after building the live filesystem."""
+ self._state = DebianBuildState.UMOUNT
+ self.doUnmounting()
+
+ def gatherResults(self):
+ """Gather the results of the build and add them to the file cache."""
+ for entry in sorted(os.listdir(self.build_path)):
+ if entry.startswith("livecd."):
+ self._slave.addWaitingFile(
+ os.path.join(self.build_path, entry))
=== added file 'lpbuildd/tests/test_livefs.py'
--- lpbuildd/tests/test_livefs.py 1970-01-01 00:00:00 +0000
+++ lpbuildd/tests/test_livefs.py 2013-11-02 20:02:43 +0000
@@ -0,0 +1,118 @@
+# Copyright 2013 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+__metaclass__ = type
+
+import os
+import shutil
+import tempfile
+
+from testtools import TestCase
+
+from lpbuildd.livefs import (
+ LiveFilesystemBuildManager,
+ LiveFilesystemBuildState,
+ )
+from lpbuildd.tests.fakeslave import FakeSlave
+
+
+class MockBuildManager(LiveFilesystemBuildManager):
+ def __init__(self, *args, **kwargs):
+ super(MockBuildManager, self).__init__(*args, **kwargs)
+ self.commands = []
+ self.iterators = []
+
+ def runSubProcess(self, path, command, iterate=None):
+ self.commands.append([path] + command)
+ if iterate is None:
+ iterate = self.iterate
+ self.iterators.append(iterate)
+ return 0
+
+
+class TestLiveFilesystemBuildManagerIteration(TestCase):
+ """Run LiveFilesystemBuildManager through its iteration steps."""
+ def setUp(self):
+ super(TestLiveFilesystemBuildManagerIteration, self).setUp()
+ self.working_dir = tempfile.mkdtemp()
+ self.addCleanup(lambda: shutil.rmtree(self.working_dir))
+ slave_dir = os.path.join(self.working_dir, "slave")
+ home_dir = os.path.join(self.working_dir, "home")
+ for dir in (slave_dir, home_dir):
+ os.mkdir(dir)
+ self.slave = FakeSlave(slave_dir)
+ self.buildid = "123"
+ self.buildmanager = MockBuildManager(self.slave, self.buildid)
+ self.buildmanager.home = home_dir
+ self.buildmanager._cachepath = self.slave._cachepath
+ self.build_dir = os.path.join(
+ home_dir, "build-%s" % self.buildid, "chroot-autobuild", "build")
+
+ def getState(self):
+ """Retrieve build manager's state."""
+ return self.buildmanager._state
+
+ def startBuild(self):
+ # The build manager's iterate() kicks off the consecutive states
+ # after INIT.
+ extra_args = {
+ "project": "ubuntu",
+ "suite": "saucy",
+ "arch_tag": "i386",
+ }
+ self.buildmanager.initiate({}, "chroot.tar.gz", extra_args)
+
+ # Skip states that are done in DebianBuildManager to the state
+ # directly before BUILD_LIVEFS.
+ self.buildmanager._state = LiveFilesystemBuildState.UPDATE
+
+ # BUILD_LIVEFS: Run the slave's payload to build the live filesystem.
+ self.buildmanager.iterate(0)
+ self.assertEqual(
+ LiveFilesystemBuildState.BUILD_LIVEFS, self.getState())
+ expected_command = [
+ "buildlivefspath", "buildlivefs", "--build-id", self.buildid,
+ "--arch", "i386", "--project", "ubuntu", "--suite", "saucy",
+ ]
+ self.assertEqual(expected_command, self.buildmanager.commands[-1])
+ self.assertEqual(
+ self.buildmanager.iterate, self.buildmanager.iterators[-1])
+ self.assertFalse(self.slave.wasCalled("chrootFail"))
+
+ def test_iterate(self):
+ # The build manager iterates a normal build from start to finish.
+ self.startBuild()
+
+ log_path = os.path.join(self.buildmanager._cachepath, "buildlog")
+ log = open(log_path, "w")
+ log.write("I am a build log.")
+ log.close()
+
+ os.makedirs(self.build_dir)
+ manifest_path = os.path.join(self.build_dir, "livecd.ubuntu.manifest")
+ manifest = open(manifest_path, "w")
+ manifest.write("I am a manifest file.")
+ manifest.close()
+
+ # After building the package, reap processes.
+ self.buildmanager.iterate(0)
+ expected_command = [
+ "processscanpath", "scan-for-processes", self.buildid,
+ ]
+ self.assertEqual(
+ LiveFilesystemBuildState.BUILD_LIVEFS, self.getState())
+ self.assertEqual(expected_command, self.buildmanager.commands[-1])
+ self.assertNotEqual(
+ self.buildmanager.iterate, self.buildmanager.iterators[-1])
+ self.assertFalse(self.slave.wasCalled("buildFail"))
+ self.assertEqual(
+ [((manifest_path,), {})], self.slave.addWaitingFile.calls)
+
+ # Control returns to the DebianBuildManager in the UMOUNT state.
+ self.buildmanager.iterateReap(self.getState(), 0)
+ expected_command = ["umountpath", "umount-chroot", self.buildid]
+ self.assertEqual(LiveFilesystemBuildState.UMOUNT, self.getState())
+ self.assertEqual(expected_command, self.buildmanager.commands[-1])
+ self.assertEqual(
+ self.buildmanager.iterate, self.buildmanager.iterators[-1])
+ self.assertFalse(self.slave.wasCalled("buildFail"))
=== modified file 'template-buildd-slave.conf'
--- template-buildd-slave.conf 2013-07-24 10:42:05 +0000
+++ template-buildd-slave.conf 2013-11-02 20:02:43 +0000
@@ -31,3 +31,6 @@
[translationtemplatesmanager]
generatepath = /usr/share/launchpad-buildd/slavebin/generate-translation-templates
resultarchive = translation-templates.tar.gz
+
+[livefilesystemmanager]
+buildlivefspath = /usr/share/launchpad-buildd/slavebin/buildlivefs
Follow ups
