launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #21768
[Merge] lp:~cjwatson/launchpad-buildd/override-sources-list-python into lp:launchpad-buildd
Colin Watson has proposed merging lp:~cjwatson/launchpad-buildd/override-sources-list-python into lp:launchpad-buildd with lp:~cjwatson/launchpad-buildd/update-chroot-python as a prerequisite.
Commit message:
Rewrite override-sources-list in Python, allowing it to have unit tests.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad-buildd/override-sources-list-python/+merge/328254
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad-buildd/override-sources-list-python into lp:launchpad-buildd.
=== modified file 'bin/override-sources-list'
--- bin/override-sources-list 2017-07-25 22:11:19 +0000
+++ bin/override-sources-list 2017-07-28 20:36:04 +0000
@@ -1,31 +1,34 @@
-#!/bin/sh
+#! /usr/bin/python -u
#
-# Copyright 2009 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
-# Buildd Slave tool to override sources.list in the chroot with a list of
-# archives
-
-# Expects build id as arg 1
-# Expects sources.list lines as subsequent args
-
-# Needs SUDO to be set to a sudo instance for passwordless access
-
-set -e
-exec 2>&1
-
-SUDO=/usr/bin/sudo
-
-BUILDID="$1"
-shift
-
-cd $HOME
-cd "build-$BUILDID/chroot-autobuild/etc/apt"
-
-echo "Overriding sources.list in build-$BUILDID"
-
-$SUDO rm -f sources.list.new
-(for archive; do
- echo "$archive"
-done) | $SUDO tee sources.list.new >/dev/null
-$SUDO mv sources.list.new sources.list
+"""Override sources.list in the chroot with a list of archives."""
+
+from __future__ import print_function
+
+__metaclass__ = type
+
+from argparse import ArgumentParser
+import sys
+
+from lpbuildd.target.chroot import ChrootSetup
+
+
+def main():
+ parser = ArgumentParser(
+ description="Override sources.list in the chroot.")
+ parser.add_argument(
+ "build_id", metavar="ID", help="update chroot for build ID")
+ parser.add_argument(
+ "archives", metavar="ARCHIVE", nargs="+", help="sources.list lines")
+ args = parser.parse_args()
+
+ print("Overriding sources.list in build-%s" % args.build_id)
+ setup = ChrootSetup(args.build_id)
+ setup.override_sources_list(args.archives)
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
=== modified file 'bin/update-debian-chroot'
--- bin/update-debian-chroot 2017-07-28 20:36:04 +0000
+++ bin/update-debian-chroot 2017-07-28 20:36:04 +0000
@@ -12,7 +12,7 @@
from argparse import ArgumentParser
import sys
-from lpbuildd.target.chroot import ChrootUpdater
+from lpbuildd.target.chroot import ChrootSetup
def main():
@@ -26,8 +26,8 @@
args = parser.parse_args()
print("Updating debian chroot for build %s" % args.build_id)
- updater = ChrootUpdater(args.build_id, args.series, args.arch)
- updater.update()
+ setup = ChrootSetup(args.build_id, args.series, args.arch)
+ setup.update()
return 0
=== modified file 'debian/changelog'
--- debian/changelog 2017-07-28 20:36:04 +0000
+++ debian/changelog 2017-07-28 20:36:04 +0000
@@ -10,6 +10,7 @@
dpkg-architecture instead.
* Rewrite update-debian-chroot in Python, allowing it to use lpbuildd.util
and to have unit tests.
+ * Rewrite override-sources-list in Python, allowing it to have unit tests.
-- Colin Watson <cjwatson@xxxxxxxxxx> Tue, 25 Jul 2017 23:07:58 +0100
=== modified file 'debian/python-lpbuildd.install'
--- debian/python-lpbuildd.install 2011-11-11 06:48:32 +0000
+++ debian/python-lpbuildd.install 2017-07-28 20:36:04 +0000
@@ -1,4 +1,5 @@
buildd-slave.tac usr/lib/launchpad-buildd
lpbuildd/*py usr/lib/launchpad-buildd/lpbuildd
lpbuildd/pottery/*py usr/lib/launchpad-buildd/lpbuildd/pottery
+lpbuildd/target/*py usr/lib/launchpad-buildd/lpbuildd/target
lpbuildd/tests/* usr/lib/launchpad-buildd/lpbuildd/tests
=== modified file 'lpbuildd/target/chroot.py'
--- lpbuildd/target/chroot.py 2017-07-28 20:36:04 +0000
+++ lpbuildd/target/chroot.py 2017-07-28 20:36:04 +0000
@@ -8,6 +8,7 @@
import os.path
import subprocess
import sys
+import tempfile
import time
from lpbuildd.util import (
@@ -16,17 +17,17 @@
)
-class ChrootUpdater:
- """Updates a chroot."""
+class ChrootSetup:
+ """Sets up a chroot."""
- def __init__(self, build_id, series, arch):
+ def __init__(self, build_id, series=None, arch=None):
self.build_id = build_id
self.series = series
self.arch = arch
self.chroot_path = os.path.join(
os.environ["HOME"], "build-" + build_id, "chroot-autobuild")
- def chroot(self, args, env=None, **kwargs):
+ def chroot(self, args, env=None, input_text=None, **kwargs):
"""Run a command in the chroot.
:param args: the command and arguments to run.
@@ -35,9 +36,32 @@
args = ["env"] + [
"%s=%s" % (key, shell_escape(value))
for key, value in env.items()] + args
- args = set_personality(args, self.arch, series=self.series)
+ if self.arch is not None:
+ args = set_personality(args, self.arch, series=self.series)
cmd = ["/usr/bin/sudo", "/usr/sbin/chroot", self.chroot_path] + args
- subprocess.check_call(cmd, **kwargs)
+ if input_text is None:
+ subprocess.check_call(cmd, **kwargs)
+ else:
+ proc = subprocess.Popen(
+ cmd, stdin=subprocess.PIPE, universal_newlines=True, **kwargs)
+ proc.communicate(input_text)
+ if proc.returncode:
+ raise subprocess.CalledProcessError(proc.returncode, cmd)
+
+ def insert_file(self, source_path, target_path,
+ owner="root", group="root", mode=0o644):
+ """Insert a file into the chroot.
+
+ :param source_path: the path to the file outside the chroot.
+ :param target_path: the path where the file should be installed
+ inside the chroot.
+ """
+ full_target_path = os.path.join(
+ self.chroot_path, target_path.lstrip("/"))
+ subprocess.check_call(
+ ["/usr/bin/sudo", "install",
+ "-o", owner, "-g", group, "-m", "%o" % mode,
+ source_path, full_target_path])
def update(self):
with open("/dev/null", "r") as devnull:
@@ -60,3 +84,10 @@
"--purge", "dist-upgrade",
]
self.chroot(upgrade_args, env=env, stdin=devnull)
+
+ def override_sources_list(self, archives):
+ with tempfile.NamedTemporaryFile() as sources_list:
+ for archive in archives:
+ print(archive, file=sources_list)
+ sources_list.flush()
+ self.insert_file(sources_list.name, "/etc/apt/sources.list")
=== modified file 'lpbuildd/target/tests/test_chroot.py'
--- lpbuildd/target/tests/test_chroot.py 2017-07-28 20:36:04 +0000
+++ lpbuildd/target/tests/test_chroot.py 2017-07-28 20:36:04 +0000
@@ -5,10 +5,12 @@
import sys
import time
+from textwrap import dedent
from fixtures import (
EnvironmentVariable,
MockPatch,
+ MockPatchObject,
)
from systemfixtures import (
FakeProcesses,
@@ -16,16 +18,30 @@
)
from testtools import TestCase
-from lpbuildd.target.chroot import ChrootUpdater
-
-
-class TestChrootUpdater(TestCase):
+from lpbuildd.target.chroot import ChrootSetup
+from lpbuildd.tests.fakeslave import FakeMethod
+
+
+class MockInsertFile(FakeMethod):
+
+ def __init__(self, *args, **kwargs):
+ super(MockInsertFile, self).__init__(*args, **kwargs)
+ self.source_bytes = None
+
+ def __call__(self, source_path, *args, **kwargs):
+ with open(source_path, "rb") as source:
+ self.source_bytes = source.read()
+ return super(MockInsertFile, self).__call__(
+ source_path, *args, **kwargs)
+
+
+class TestChrootSetup(TestCase):
def test_chroot(self):
self.useFixture(EnvironmentVariable("HOME", "/expected/home"))
processes_fixture = self.useFixture(FakeProcesses())
processes_fixture.add(lambda _: {}, name="/usr/bin/sudo")
- ChrootUpdater("1", "xenial", "amd64").chroot(
+ ChrootSetup("1", "xenial", "amd64").chroot(
["apt-get", "update"], env={"LANG": "C"})
expected_args = [
@@ -37,13 +53,33 @@
expected_args,
[proc._args["args"] for proc in processes_fixture.procs])
+ def test_insert_file(self):
+ self.useFixture(EnvironmentVariable("HOME", "/expected/home"))
+ processes_fixture = self.useFixture(FakeProcesses())
+ processes_fixture.add(lambda _: {}, name="/usr/bin/sudo")
+ source_path = "/path/to/source"
+ target_path = "/path/to/target"
+ ChrootSetup("1", "xenial", "amd64").insert_file(
+ source_path, target_path)
+
+ expected_target_path = (
+ "/expected/home/build-1/chroot-autobuild/path/to/target")
+ expected_args = [
+ ["/usr/bin/sudo", "install",
+ "-o", "root", "-g", "root", "-m", "644",
+ source_path, expected_target_path],
+ ]
+ self.assertEqual(
+ expected_args,
+ [proc._args["args"] for proc in processes_fixture.procs])
+
def test_update_succeeds(self):
self.useFixture(EnvironmentVariable("HOME", "/expected/home"))
processes_fixture = self.useFixture(FakeProcesses())
processes_fixture.add(lambda _: {}, name="/usr/bin/sudo")
self.useFixture(FakeTime())
start_time = time.time()
- ChrootUpdater("1", "xenial", "amd64").update()
+ ChrootSetup("1", "xenial", "amd64").update()
apt_get_args = [
"/usr/bin/sudo", "/usr/sbin/chroot",
@@ -75,7 +111,7 @@
mock_print = self.useFixture(MockPatch("__builtin__.print")).mock
self.useFixture(FakeTime())
start_time = time.time()
- ChrootUpdater("1", "xenial", "amd64").update()
+ ChrootSetup("1", "xenial", "amd64").update()
apt_get_args = [
"/usr/bin/sudo", "/usr/sbin/chroot",
@@ -100,3 +136,21 @@
mock_print.assert_called_once_with(
"Waiting 15 seconds and trying again ...", file=sys.stderr)
self.assertEqual(start_time + 15, time.time())
+
+ def test_override_sources_list(self):
+ self.useFixture(EnvironmentVariable("HOME", "/expected/home"))
+ setup = ChrootSetup("1")
+ mock_insert_file = self.useFixture(
+ MockPatchObject(setup, "insert_file", new=MockInsertFile())).mock
+ setup.override_sources_list([
+ "deb http://archive.ubuntu.com/ubuntu xenial main",
+ "deb http://ppa.launchpad.net/launchpad/ppa/ubuntu xenial main",
+ ])
+
+ self.assertEqual(dedent("""\
+ deb http://archive.ubuntu.com/ubuntu xenial main
+ deb http://ppa.launchpad.net/launchpad/ppa/ubuntu xenial main
+ """).encode("UTF-8"), mock_insert_file.source_bytes)
+ self.assertEqual(
+ ("/etc/apt/sources.list",),
+ mock_insert_file.extract_args()[0][1:])
