launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #23217
[Merge] lp:~cjwatson/launchpad-buildd/image-type into lp:launchpad-buildd
Colin Watson has proposed merging lp:~cjwatson/launchpad-buildd/image-type into lp:launchpad-buildd.
Commit message:
Allow the LXD backend to accept a LXD image instead of a chroot tarball, skipping the conversion step.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad-buildd/image-type/+merge/361633
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad-buildd/image-type into lp:launchpad-buildd.
=== modified file 'debian/changelog'
--- debian/changelog 2018-11-07 11:48:18 +0000
+++ debian/changelog 2019-01-10 18:17:59 +0000
@@ -2,6 +2,8 @@
[ Colin Watson ]
* Run all tests at package build time, not just those in lpbuildd.tests.
+ * Allow the LXD backend to accept a LXD image instead of a chroot tarball,
+ skipping the conversion step.
[ Tobias Koch ]
* Update LXD backend to work with LXD 3.
=== modified file 'lpbuildd/slave.py'
--- lpbuildd/slave.py 2018-05-02 09:00:10 +0000
+++ lpbuildd/slave.py 2019-01-10 18:17:59 +0000
@@ -182,7 +182,9 @@
def doUnpack(self):
"""Unpack the build chroot."""
- self.runTargetSubProcess("unpack-chroot", self._chroottarfile)
+ self.runTargetSubProcess(
+ "unpack-chroot", "--image-type", self.image_type,
+ self._chroottarfile)
def doReapProcesses(self, state, notify=True):
"""Reap any processes left lying around in the chroot."""
@@ -242,6 +244,7 @@
self._buildid, f))
self._chroottarfile = self._slave.cachePath(chroot)
+ self.image_type = extra_args.get('image_type', 'chroot')
self.series = extra_args['series']
self.arch_tag = extra_args.get('arch_tag', self._slave.getArch())
self.fast_cleanup = extra_args.get('fast_cleanup', False)
=== modified file 'lpbuildd/target/backend.py'
--- lpbuildd/target/backend.py 2017-11-10 20:55:33 +0000
+++ lpbuildd/target/backend.py 2019-01-10 18:17:59 +0000
@@ -22,8 +22,8 @@
self.arch = arch
self.build_path = os.path.join(os.environ["HOME"], "build-" + build_id)
- def create(self, tarball_path):
- """Create the backend based on a chroot tarball.
+ def create(self, image_path, image_type):
+ """Create the backend based on a base image.
This puts the backend into a state where it is ready to be started.
"""
=== modified file 'lpbuildd/target/chroot.py'
--- lpbuildd/target/chroot.py 2018-06-12 23:23:13 +0000
+++ lpbuildd/target/chroot.py 2019-01-10 18:17:59 +0000
@@ -28,10 +28,13 @@
super(Chroot, self).__init__(build_id, series=series, arch=arch)
self.chroot_path = os.path.join(self.build_path, "chroot-autobuild")
- def create(self, tarball_path):
+ def create(self, image_path, image_type):
"""See `Backend`."""
- subprocess.check_call(
- ["sudo", "tar", "-C", self.build_path, "-xf", tarball_path])
+ if image_type == "chroot":
+ subprocess.check_call(
+ ["sudo", "tar", "-C", self.build_path, "-xf", image_path])
+ else:
+ raise ValueError("Unhandled image type: %s" % image_type)
def start(self):
"""See `Backend`."""
=== modified file 'lpbuildd/target/lifecycle.py'
--- lpbuildd/target/lifecycle.py 2017-08-22 14:48:06 +0000
+++ lpbuildd/target/lifecycle.py 2019-01-10 18:17:59 +0000
@@ -23,11 +23,13 @@
@classmethod
def add_arguments(cls, parser):
super(Create, cls).add_arguments(parser)
- parser.add_argument("tarball_path", help="path to chroot tarball")
+ parser.add_argument(
+ "--image-type", default="chroot", help="base image type")
+ parser.add_argument("image_path", help="path to base image")
def run(self):
logger.info("Creating target for build %s", self.args.build_id)
- self.backend.create(self.args.tarball_path)
+ self.backend.create(self.args.image_path, self.args.image_type)
return 0
=== modified file 'lpbuildd/target/lxd.py'
--- lpbuildd/target/lxd.py 2018-10-25 09:54:49 +0000
+++ lpbuildd/target/lxd.py 2019-01-10 18:17:59 +0000
@@ -170,22 +170,29 @@
if fileptr is not None:
fileptr.close()
- def create(self, tarball_path):
+ def create(self, image_path, image_type):
"""See `Backend`."""
self.remove_image()
# This is a lot of data to shuffle around in Python, but there
# doesn't currently seem to be any way to ask pylxd to ask lxd to
# import an image from a file on disk.
- with io.BytesIO() as target_file:
- with tarfile.open(name=tarball_path, mode="r") as source_tarball:
- with tarfile.open(
- fileobj=target_file, mode="w") as target_tarball:
- self._convert(source_tarball, target_tarball)
-
- image = self.client.images.create(
- target_file.getvalue(), wait=True)
- image.add_alias(self.alias, self.alias)
+ if image_type == "chroot":
+ with io.BytesIO() as target_file:
+ with tarfile.open(name=image_path, mode="r") as source_tarball:
+ with tarfile.open(
+ fileobj=target_file, mode="w") as target_tarball:
+ self._convert(source_tarball, target_tarball)
+
+ image = self.client.images.create(
+ target_file.getvalue(), wait=True)
+ elif image_type == "lxd":
+ with open(image_path, "rb") as image_file:
+ image = self.client.images.create(image_file.read(), wait=True)
+ else:
+ raise ValueError("Unhandled image type: %s" % image_type)
+
+ image.add_alias(self.alias, self.alias)
@property
def sys_dir(self):
=== modified file 'lpbuildd/target/tests/test_chroot.py'
--- lpbuildd/target/tests/test_chroot.py 2018-06-12 23:23:13 +0000
+++ lpbuildd/target/tests/test_chroot.py 2019-01-10 18:17:59 +0000
@@ -35,7 +35,7 @@
self.useFixture(EnvironmentVariable("HOME", "/expected/home"))
processes_fixture = self.useFixture(FakeProcesses())
processes_fixture.add(lambda _: {}, name="sudo")
- Chroot("1", "xenial", "amd64").create("/path/to/tarball")
+ Chroot("1", "xenial", "amd64").create("/path/to/tarball", "chroot")
expected_args = [
["sudo", "tar", "-C", "/expected/home/build-1",
=== modified file 'lpbuildd/target/tests/test_lifecycle.py'
--- lpbuildd/target/tests/test_lifecycle.py 2017-08-22 15:09:09 +0000
+++ lpbuildd/target/tests/test_lifecycle.py 2019-01-10 18:17:59 +0000
@@ -24,7 +24,20 @@
create = parse_args(args=args).operation
self.assertEqual(0, create.run())
self.assertEqual(
- [(("/path/to/tarball",), {})], create.backend.create.calls)
+ [(("/path/to/tarball", "chroot"), {})],
+ create.backend.create.calls)
+
+ def test_image_type(self):
+ args = [
+ "unpack-chroot",
+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+ "--image-type", "lxd", "/path/to/tarball",
+ ]
+ create = parse_args(args=args).operation
+ self.assertEqual(0, create.run())
+ self.assertEqual(
+ [(("/path/to/tarball", "lxd"), {})],
+ create.backend.create.calls)
class TestStart(TestCase):
=== modified file 'lpbuildd/target/tests/test_lxd.py'
--- lpbuildd/target/tests/test_lxd.py 2018-10-25 09:54:49 +0000
+++ lpbuildd/target/tests/test_lxd.py 2019-01-10 18:17:59 +0000
@@ -12,6 +12,7 @@
import os.path
import tarfile
from textwrap import dedent
+import time
try:
from unittest import mock
except ImportError:
@@ -101,6 +102,32 @@
with tarfile.open(output_path, "w:bz2") as tar:
tar.add(source, arcname="chroot-autobuild")
+ def make_lxd_image(self, output_path):
+ source = self.useFixture(TempDir()).path
+ hello = os.path.join(source, "bin", "hello")
+ os.mkdir(os.path.dirname(hello))
+ with open(hello, "w") as f:
+ f.write("hello\n")
+ os.fchmod(f.fileno(), 0o755)
+ metadata = {
+ "architecture": "x86_64",
+ "creation_date": time.time(),
+ "properties": {
+ "os": "Ubuntu",
+ "series": "xenial",
+ "architecture": "amd64",
+ "description": "Launchpad chroot for Ubuntu xenial (amd64)",
+ },
+ }
+ metadata_yaml = json.dumps(
+ metadata, sort_keys=True, indent=4, separators=(",", ": "),
+ ensure_ascii=False).encode("UTF-8") + b"\n"
+ with tarfile.open(output_path, "w:gz") as tar:
+ metadata_file = tarfile.TarInfo(name="metadata.yaml")
+ metadata_file.size = len(metadata_yaml)
+ tar.addfile(metadata_file, io.BytesIO(metadata_yaml))
+ tar.add(source, arcname="rootfs")
+
def test_convert(self):
tmp = self.useFixture(TempDir()).path
source_tarball_path = os.path.join(tmp, "source.tar.bz2")
@@ -136,7 +163,7 @@
self.assertThat(hello, FileContains("hello\n"))
self.assertThat(hello, HasPermissions("0755"))
- def test_create(self):
+ def test_create_from_chroot(self):
tmp = self.useFixture(TempDir()).path
source_tarball_path = os.path.join(tmp, "source.tar.bz2")
self.make_chroot_tarball(source_tarball_path)
@@ -145,7 +172,26 @@
client.images.all.return_value = []
image = mock.MagicMock()
client.images.create.return_value = image
- LXD("1", "xenial", "amd64").create(source_tarball_path)
+ LXD("1", "xenial", "amd64").create(source_tarball_path, "chroot")
+
+ client.images.create.assert_called_once_with(mock.ANY, wait=True)
+ with io.BytesIO(client.images.create.call_args[0][0]) as f:
+ with tarfile.open(fileobj=f) as tar:
+ with closing(tar.extractfile("rootfs/bin/hello")) as hello:
+ self.assertEqual("hello\n", hello.read())
+ image.add_alias.assert_called_once_with(
+ "lp-xenial-amd64", "lp-xenial-amd64")
+
+ def test_create_from_lxd(self):
+ tmp = self.useFixture(TempDir()).path
+ source_image_path = os.path.join(tmp, "source.tar.gz")
+ self.make_lxd_image(source_image_path)
+ self.useFixture(MockPatch("pylxd.Client"))
+ client = pylxd.Client()
+ client.images.all.return_value = []
+ image = mock.MagicMock()
+ client.images.create.return_value = image
+ LXD("1", "xenial", "amd64").create(source_image_path, "lxd")
client.images.create.assert_called_once_with(mock.ANY, wait=True)
with io.BytesIO(client.images.create.call_args[0][0]) as f:
=== modified file 'lpbuildd/tests/test_debian.py'
--- lpbuildd/tests/test_debian.py 2018-05-02 09:00:10 +0000
+++ lpbuildd/tests/test_debian.py 2019-01-10 18:17:59 +0000
@@ -109,6 +109,7 @@
'unpack-chroot',
'--backend=chroot', '--series=xenial', '--arch=amd64',
self.buildid,
+ '--image-type', 'chroot',
os.path.join(self.buildmanager._cachepath, 'chroot.tar.gz')],
None),
self.buildmanager.commands[-1])
@@ -223,6 +224,7 @@
'unpack-chroot',
'--backend=chroot', '--series=xenial', '--arch=amd64',
self.buildid,
+ '--image-type', 'chroot',
os.path.join(self.buildmanager._cachepath, 'chroot.tar.gz')],
None),
self.buildmanager.commands[-1])
@@ -351,6 +353,7 @@
'unpack-chroot',
'--backend=chroot', '--series=xenial', '--arch=amd64',
self.buildid,
+ '--image-type', 'chroot',
os.path.join(self.buildmanager._cachepath, 'chroot.tar.gz')],
None),
self.buildmanager.commands[-1])
@@ -421,3 +424,27 @@
self.assertTrue(self.slave.wasCalled('buildOK'))
self.assertTrue(self.slave.wasCalled('buildComplete'))
+ def test_iterate_lxd(self):
+ # The build manager passes the image_type argument through to
+ # unpack-chroot.
+ self.buildmanager.backend_name = 'lxd'
+ extra_args = {
+ 'image_type': 'lxd',
+ 'arch_tag': 'amd64',
+ 'series': 'xenial',
+ }
+ self.startBuild(extra_args)
+
+ self.buildmanager.iterate(0)
+ self.assertEqual(DebianBuildState.UNPACK, self.getState())
+ self.assertEqual(
+ (['sharepath/slavebin/in-target', 'in-target',
+ 'unpack-chroot',
+ '--backend=lxd', '--series=xenial', '--arch=amd64',
+ self.buildid,
+ '--image-type', 'lxd',
+ os.path.join(self.buildmanager._cachepath, 'chroot.tar.gz')],
+ None),
+ self.buildmanager.commands[-1])
+ self.assertEqual(
+ self.buildmanager.iterate, self.buildmanager.iterators[-1])
