launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #21767
[Merge] lp:~cjwatson/launchpad-buildd/update-chroot-python into lp:launchpad-buildd
Colin Watson has proposed merging lp:~cjwatson/launchpad-buildd/update-chroot-python into lp:launchpad-buildd with lp:~cjwatson/launchpad-buildd/reduce-arch-hardcoding as a prerequisite.
Commit message:
Rewrite update-debian-chroot in Python, allowing it to use lpbuildd.util
and to have unit tests.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad-buildd/update-chroot-python/+merge/328223
I put the guts of the rewritten script in a new lpbuildd.target namespace to keep things organised. My intent is that everything that runs in the target (chroot or eventually LXD) should end up in there.
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad-buildd/update-chroot-python into lp:launchpad-buildd.
=== modified file 'bin/update-debian-chroot'
--- bin/update-debian-chroot 2017-07-26 12:01:39 +0000
+++ bin/update-debian-chroot 2017-07-28 14:06:24 +0000
@@ -1,54 +1,35 @@
-#!/bin/sh
+#! /usr/bin/python -u
#
-# Copyright 2009 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
-# Buildd Slave tool to update a debian chroot
-
-# Expects build id as arg 1, makes build-id to contain the build
-
-# Needs SUDO to be set to a sudo instance for passwordless access
-
-SUDO=/usr/bin/sudo
-CHROOT=/usr/sbin/chroot
-APTGET=/usr/bin/apt-get
-BUILDID="$1"
-ARCHITECTURETAG="$2"
-ROOT=$HOME/build-$BUILDID/chroot-autobuild
-OPTIONS="-o DPkg::Options::=--force-confold"
-
-set -e
-
-exec 2>&1
-
-echo "Updating debian chroot for build $BUILDID"
-
-UNAME26=""
-case $SUITE in
- hardy*|lucid*|maverick*|natty*|oneiric*|precise*)
- if setarch --help | grep -q uname-2.6; then
- UNAME26="--uname-2.6"
- fi
- ;;
-esac
-
-case $ARCHITECTURETAG in
- armel|armhf|hppa|i386|lpia|mips|mipsel|powerpc|s390|sparc)
- CHROOT="linux32 $UNAME26 $CHROOT"
- ;;
- alpha|amd64|arm64|hppa64|ia64|ppc64|ppc64el|s390x|sparc64|x32)
- CHROOT="linux64 $UNAME26 $CHROOT"
- ;;
-esac
-
-export LANG=C
-export DEBIAN_FRONTEND=noninteractive
-export TTY=unknown
-
-if ! $SUDO $CHROOT $ROOT $APTGET -uy update < /dev/null; then
- echo "Waiting 15 seconds and trying again ..." >&2
- sleep 15
- $SUDO $CHROOT $ROOT $APTGET -uy update < /dev/null
-fi
-$SUDO $CHROOT $ROOT $APTGET $OPTIONS -uy --purge dist-upgrade < /dev/null
-
+"""Update a Debian-style chroot."""
+
+from __future__ import print_function
+
+__metaclass__ = type
+
+from argparse import ArgumentParser
+import sys
+
+from lpbuildd.target.chroot import ChrootUpdater
+
+
+def main():
+ parser = ArgumentParser(description="Update a Debian-style chroot.")
+ parser.add_argument(
+ "--series", metavar="SERIES", help="update chroot for series SERIES")
+ parser.add_argument(
+ "--arch", metavar="ARCH", help="update chroot for architecture ARCH")
+ parser.add_argument(
+ "build_id", metavar="ID", help="update chroot for build ID")
+ args = parser.parse_args()
+
+ print("Updating debian chroot for build %s" % args.build_id)
+ updater = ChrootUpdater(args.build_id, args.series, args.arch)
+ updater.update()
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
=== modified file 'debian/changelog'
--- debian/changelog 2017-07-28 14:06:23 +0000
+++ debian/changelog 2017-07-28 14:06:24 +0000
@@ -8,6 +8,8 @@
real hardware for a while.
* Remove most architecture hardcoding from lpbuildd.util, relying on
dpkg-architecture instead.
+ * Rewrite update-debian-chroot in Python, allowing it to use lpbuildd.util
+ and to have unit tests.
-- Colin Watson <cjwatson@xxxxxxxxxx> Tue, 25 Jul 2017 23:07:58 +0100
=== modified file 'debian/control'
--- debian/control 2017-07-28 13:22:05 +0000
+++ debian/control 2017-07-28 14:06:24 +0000
@@ -3,7 +3,11 @@
Priority: extra
Maintainer: Adam Conrad <adconrad@xxxxxxxxxx>
Standards-Version: 3.9.5
+<<<<<<< TREE
Build-Depends: debhelper (>= 7.0.50~), apt-utils, bzr, intltool, python, python-apt, python-debian, python-fixtures, python-testtools, python-twisted, python-txfixtures, python-zope.interface
+=======
+Build-Depends: debhelper (>= 7.0.50~), bzr, intltool, python, python-apt, python-debian, python-fixtures, python-mock, python-systemfixtures, python-testtools, python-twisted, python-txfixtures, python-zope.interface
+>>>>>>> MERGE-SOURCE
Package: launchpad-buildd
Section: misc
=== modified file 'lpbuildd/debian.py'
--- lpbuildd/debian.py 2017-04-26 12:24:32 +0000
+++ lpbuildd/debian.py 2017-07-28 14:06:24 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009, 2010 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
# Authors: Daniel Silverstone <daniel.silverstone@xxxxxxxxxxxxx>
@@ -52,7 +52,7 @@
def initiate(self, files, chroot, extra_args):
"""Initiate a build with a given set of files and chroot."""
-
+ self.series = extra_args['series']
self.arch_tag = extra_args.get('arch_tag', self._slave.getArch())
self.sources_list = extra_args.get('archives')
self.trusted_keys = extra_args.get('trusted_keys')
@@ -80,7 +80,10 @@
"""Perform the chroot upgrade."""
self.runSubProcess(
self._updatepath,
- ["update-debian-chroot", self._buildid, self.arch_tag])
+ ["update-debian-chroot",
+ "--series", self.series,
+ "--arch", self.arch_tag,
+ self._buildid])
def doRunBuild(self):
"""Run the main build process.
=== modified file 'lpbuildd/livefs.py'
--- lpbuildd/livefs.py 2015-05-11 05:39:25 +0000
+++ lpbuildd/livefs.py 2017-07-28 14:06:24 +0000
@@ -1,4 +1,4 @@
-# Copyright 2013 Canonical Ltd. This software is licensed under the
+# Copyright 2013-2017 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
__metaclass__ = type
@@ -41,7 +41,6 @@
self.subarch = extra_args.get("subarch")
self.project = extra_args["project"]
self.subproject = extra_args.get("subproject")
- self.series = extra_args["series"]
self.pocket = extra_args["pocket"]
self.datestamp = extra_args.get("datestamp")
self.image_format = extra_args.get("image_format")
=== added directory 'lpbuildd/target'
=== added file 'lpbuildd/target/__init__.py'
=== added file 'lpbuildd/target/chroot.py'
--- lpbuildd/target/chroot.py 1970-01-01 00:00:00 +0000
+++ lpbuildd/target/chroot.py 2017-07-28 14:06:24 +0000
@@ -0,0 +1,62 @@
+# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from __future__ import print_function
+
+__metaclass__ = type
+
+import os.path
+import subprocess
+import sys
+import time
+
+from lpbuildd.util import (
+ set_personality,
+ shell_escape,
+ )
+
+
+class ChrootUpdater:
+ """Updates a chroot."""
+
+ def __init__(self, build_id, series, arch):
+ self.build_id = build_id
+ self.series = series
+ self.arch = arch
+ self.chroot_path = os.path.join(
+ os.environ["HOME"], "build-" + build_id, "chroot-autobuild")
+
+ def chroot(self, args, env=None, **kwargs):
+ """Run a command in the chroot.
+
+ :param args: the command and arguments to run.
+ """
+ if env:
+ args = ["env"] + [
+ "%s=%s" % (key, shell_escape(value))
+ for key, value in env.items()] + args
+ args = set_personality(args, self.arch, series=self.series)
+ cmd = ["/usr/bin/sudo", "/usr/sbin/chroot", self.chroot_path] + args
+ subprocess.check_call(cmd, **kwargs)
+
+ def update(self):
+ with open("/dev/null", "r") as devnull:
+ env = {
+ "LANG": "C",
+ "DEBIAN_FRONTEND": "noninteractive",
+ "TTY": "unknown",
+ }
+ apt_get = "/usr/bin/apt-get"
+ update_args = [apt_get, "-uy", "update"]
+ try:
+ self.chroot(update_args, env=env, stdin=devnull)
+ except subprocess.CalledProcessError:
+ print(
+ "Waiting 15 seconds and trying again ...", file=sys.stderr)
+ time.sleep(15)
+ self.chroot(update_args, env=env, stdin=devnull)
+ upgrade_args = [
+ apt_get, "-o", "DPkg::Options::=--force-confold", "-uy",
+ "--purge", "dist-upgrade",
+ ]
+ self.chroot(upgrade_args, env=env, stdin=devnull)
=== added directory 'lpbuildd/target/tests'
=== added file 'lpbuildd/target/tests/__init__.py'
=== added file 'lpbuildd/target/tests/test_chroot.py'
--- lpbuildd/target/tests/test_chroot.py 1970-01-01 00:00:00 +0000
+++ lpbuildd/target/tests/test_chroot.py 2017-07-28 14:06:24 +0000
@@ -0,0 +1,102 @@
+# Copyright 2017 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+__metaclass__ = type
+
+import sys
+import time
+
+from fixtures import (
+ EnvironmentVariable,
+ MockPatch,
+ )
+from systemfixtures import (
+ FakeProcesses,
+ FakeTime,
+ )
+from testtools import TestCase
+
+from lpbuildd.target.chroot import ChrootUpdater
+
+
+class TestChrootUpdater(TestCase):
+
+ def test_chroot(self):
+ self.useFixture(EnvironmentVariable("HOME", "/expected/home"))
+ processes_fixture = self.useFixture(FakeProcesses())
+ processes_fixture.add(lambda _: {}, name="/usr/bin/sudo")
+ ChrootUpdater("1", "xenial", "amd64").chroot(
+ ["apt-get", "update"], env={"LANG": "C"})
+
+ expected_args = [
+ ["/usr/bin/sudo", "/usr/sbin/chroot",
+ "/expected/home/build-1/chroot-autobuild",
+ "linux64", "env", "LANG=C", "apt-get", "update"],
+ ]
+ self.assertEqual(
+ expected_args,
+ [proc._args["args"] for proc in processes_fixture.procs])
+
+ def test_update_succeeds(self):
+ self.useFixture(EnvironmentVariable("HOME", "/expected/home"))
+ processes_fixture = self.useFixture(FakeProcesses())
+ processes_fixture.add(lambda _: {}, name="/usr/bin/sudo")
+ self.useFixture(FakeTime())
+ start_time = time.time()
+ ChrootUpdater("1", "xenial", "amd64").update()
+
+ apt_get_args = [
+ "/usr/bin/sudo", "/usr/sbin/chroot",
+ "/expected/home/build-1/chroot-autobuild",
+ "linux64", "env",
+ "LANG=C",
+ "DEBIAN_FRONTEND=noninteractive",
+ "TTY=unknown",
+ "/usr/bin/apt-get",
+ ]
+ expected_args = [
+ apt_get_args + ["-uy", "update"],
+ apt_get_args + [
+ "-o", "DPkg::Options::=--force-confold", "-uy", "--purge",
+ "dist-upgrade",
+ ],
+ ]
+ self.assertEqual(
+ expected_args,
+ [proc._args["args"] for proc in processes_fixture.procs])
+ self.assertEqual(start_time, time.time())
+
+ def test_update_first_run_fails(self):
+ self.useFixture(EnvironmentVariable("HOME", "/expected/home"))
+ processes_fixture = self.useFixture(FakeProcesses())
+ apt_get_proc_infos = iter([{"returncode": 1}, {}, {}])
+ processes_fixture.add(
+ lambda _: next(apt_get_proc_infos), name="/usr/bin/sudo")
+ mock_print = self.useFixture(MockPatch("__builtin__.print")).mock
+ self.useFixture(FakeTime())
+ start_time = time.time()
+ ChrootUpdater("1", "xenial", "amd64").update()
+
+ apt_get_args = [
+ "/usr/bin/sudo", "/usr/sbin/chroot",
+ "/expected/home/build-1/chroot-autobuild",
+ "linux64", "env",
+ "LANG=C",
+ "DEBIAN_FRONTEND=noninteractive",
+ "TTY=unknown",
+ "/usr/bin/apt-get",
+ ]
+ expected_args = [
+ apt_get_args + ["-uy", "update"],
+ apt_get_args + ["-uy", "update"],
+ apt_get_args + [
+ "-o", "DPkg::Options::=--force-confold", "-uy", "--purge",
+ "dist-upgrade",
+ ],
+ ]
+ self.assertEqual(
+ expected_args,
+ [proc._args["args"] for proc in processes_fixture.procs])
+ mock_print.assert_called_once_with(
+ "Waiting 15 seconds and trying again ...", file=sys.stderr)
+ self.assertEqual(start_time + 15, time.time())
=== modified file 'lpbuildd/tests/test_binarypackage.py'
--- lpbuildd/tests/test_binarypackage.py 2016-12-02 14:20:28 +0000
+++ lpbuildd/tests/test_binarypackage.py 2017-07-28 14:06:24 +0000
@@ -1,4 +1,4 @@
-# Copyright 2013-2016 Canonical Ltd. This software is licensed under the
+# Copyright 2013-2017 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
__metaclass__ = type
@@ -95,7 +95,7 @@
# after INIT.
self.buildmanager.initiate(
{'foo_1.dsc': dscname}, 'chroot.tar.gz',
- {'distribution': 'ubuntu', 'suite': 'warty',
+ {'distribution': 'ubuntu', 'series': 'warty', 'suite': 'warty',
'ogrecomponent': 'main'})
os.makedirs(self.chrootdir)
@@ -170,7 +170,7 @@
# appropriately, and does not pass DEB_BUILD_OPTIONS.
self.buildmanager.initiate(
{'foo_1.dsc': ''}, 'chroot.tar.gz',
- {'distribution': 'ubuntu', 'suite': 'warty',
+ {'distribution': 'ubuntu', 'series': 'warty', 'suite': 'warty',
'ogrecomponent': 'main', 'archive_purpose': 'PRIMARY',
'build_debug_symbols': True})
os.makedirs(self.chrootdir)
@@ -198,7 +198,7 @@
# appropriately, and passes DEB_BUILD_OPTIONS=noautodbgsym.
self.buildmanager.initiate(
{'foo_1.dsc': ''}, 'chroot.tar.gz',
- {'distribution': 'ubuntu', 'suite': 'warty',
+ {'distribution': 'ubuntu', 'series': 'warty', 'suite': 'warty',
'ogrecomponent': 'main', 'archive_purpose': 'PRIMARY',
'build_debug_symbols': False})
os.makedirs(self.chrootdir)
@@ -281,7 +281,7 @@
# pretends that it was terminated by a signal.
self.buildmanager.initiate(
{'foo_1.dsc': ''}, 'chroot.tar.gz',
- {'distribution': 'ubuntu', 'suite': 'warty',
+ {'distribution': 'ubuntu', 'series': 'warty', 'suite': 'warty',
'ogrecomponent': 'main'})
self.buildmanager.abort()
@@ -493,7 +493,7 @@
# returns those relevant to the current architecture.
self.buildmanager.initiate(
{'foo_1.dsc': ''}, 'chroot.tar.gz',
- {'distribution': 'ubuntu', 'suite': 'warty',
+ {'distribution': 'ubuntu', 'series': 'warty', 'suite': 'warty',
'ogrecomponent': 'main', 'arch_tag': 'i386'})
self.assertEqual(
"foo (>= 1)",
@@ -507,7 +507,7 @@
# from the unsatisfied build-dependencies it returns.
self.buildmanager.initiate(
{'foo_1.dsc': ''}, 'chroot.tar.gz',
- {'distribution': 'ubuntu', 'suite': 'warty',
+ {'distribution': 'ubuntu', 'series': 'warty', 'suite': 'warty',
'ogrecomponent': 'main', 'arch_tag': 'i386'})
self.assertEqual(
"foo",
@@ -521,7 +521,7 @@
# that evaluate to true when no build profiles are active.
self.buildmanager.initiate(
{'foo_1.dsc': ''}, 'chroot.tar.gz',
- {'distribution': 'ubuntu', 'suite': 'warty',
+ {'distribution': 'ubuntu', 'series': 'warty', 'suite': 'warty',
'ogrecomponent': 'main', 'arch_tag': 'i386'})
self.assertEqual(
"foo",
=== modified file 'lpbuildd/tests/test_debian.py'
--- lpbuildd/tests/test_debian.py 2017-04-26 12:24:32 +0000
+++ lpbuildd/tests/test_debian.py 2017-07-28 14:06:24 +0000
@@ -98,6 +98,7 @@
'archives': [
'deb http://ppa.launchpad.dev/owner/name/ubuntu xenial main',
],
+ 'series': 'xenial',
}
self.startBuild(extra_args)
@@ -136,7 +137,8 @@
self.assertEqual(DebianBuildState.UPDATE, self.getState())
self.assertEqual(
(['sharepath/slavebin/update-debian-chroot',
- 'update-debian-chroot', self.buildid, 'amd64'],
+ 'update-debian-chroot', '--series', 'xenial', '--arch', 'amd64',
+ self.buildid],
None),
self.buildmanager.commands[-1])
self.assertEqual(
@@ -194,6 +196,7 @@
'archives': [
'deb http://ppa.launchpad.dev/owner/name/ubuntu xenial main',
],
+ 'series': 'xenial',
'trusted_keys': [base64.b64encode(b'key material')],
}
self.startBuild(extra_args)
@@ -243,7 +246,8 @@
self.assertEqual(DebianBuildState.UPDATE, self.getState())
self.assertEqual(
(['sharepath/slavebin/update-debian-chroot',
- 'update-debian-chroot', self.buildid, 'amd64'],
+ 'update-debian-chroot', '--series', 'xenial', '--arch', 'amd64',
+ self.buildid],
None),
self.buildmanager.commands[-1])
self.assertEqual(
=== modified file 'lpbuildd/tests/test_snap.py'
--- lpbuildd/tests/test_snap.py 2017-04-03 12:30:15 +0000
+++ lpbuildd/tests/test_snap.py 2017-07-28 14:06:24 +0000
@@ -1,4 +1,4 @@
-# Copyright 2015-2016 Canonical Ltd. This software is licensed under the
+# Copyright 2015-2017 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
__metaclass__ = type
@@ -56,6 +56,7 @@
# The build manager's iterate() kicks off the consecutive states
# after INIT.
extra_args = {
+ "series": "xenial",
"arch_tag": "i386",
"name": "test-snap",
"git_repository": "https://git.launchpad.dev/~example/+git/snap";,
=== modified file 'lpbuildd/tests/test_sourcepackagerecipe.py'
--- lpbuildd/tests/test_sourcepackagerecipe.py 2016-01-05 14:05:42 +0000
+++ lpbuildd/tests/test_sourcepackagerecipe.py 2017-07-28 14:06:24 +0000
@@ -1,4 +1,4 @@
-# Copyright 2013 Canonical Ltd. This software is licensed under the
+# Copyright 2013-2017 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
__metaclass__ = type
@@ -61,6 +61,7 @@
'recipe_text': dedent("""\
# bzr-builder format 0.2 deb-version {debupstream}-0~{revno}
http://bazaar.launchpad.dev/~ppa-user/+junk/wakeonlan""";),
+ 'series': 'maverick',
'suite': 'maverick',
'ogrecomponent': 'universe',
'author_name': 'Steve\u1234',
=== modified file 'lpbuildd/tests/test_translationtemplatesbuildmanager.py'
--- lpbuildd/tests/test_translationtemplatesbuildmanager.py 2015-05-11 05:55:24 +0000
+++ lpbuildd/tests/test_translationtemplatesbuildmanager.py 2017-07-28 14:06:24 +0000
@@ -1,4 +1,4 @@
-# Copyright 2010, 2011 Canonical Ltd. This software is licensed under the
+# Copyright 2010-2017 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
__metaclass__ = type
@@ -56,7 +56,8 @@
url = 'lp:~my/branch'
# The build manager's iterate() kicks off the consecutive states
# after INIT.
- self.buildmanager.initiate({}, 'chroot.tar.gz', {'branch_url': url})
+ self.buildmanager.initiate(
+ {}, 'chroot.tar.gz', {'series': 'xenial', 'branch_url': url})
# Skip states that are done in DebianBuildManager to the state
# directly before INSTALL.
@@ -132,7 +133,8 @@
url = 'lp:~my/branch'
# The build manager's iterate() kicks off the consecutive states
# after INIT.
- self.buildmanager.initiate({}, 'chroot.tar.gz', {'branch_url': url})
+ self.buildmanager.initiate(
+ {}, 'chroot.tar.gz', {'series': 'xenial', 'branch_url': url})
# Skip states to the INSTALL state.
self.buildmanager._state = TranslationTemplatesBuildState.INSTALL
@@ -154,7 +156,8 @@
url = 'lp:~my/branch'
# The build manager's iterate() kicks off the consecutive states
# after INIT.
- self.buildmanager.initiate({}, 'chroot.tar.gz', {'branch_url': url})
+ self.buildmanager.initiate(
+ {}, 'chroot.tar.gz', {'series': 'xenial', 'branch_url': url})
# Skip states to the INSTALL state.
self.buildmanager._state = TranslationTemplatesBuildState.GENERATE
