launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #32858
[Merge] ~mwhudson/launchpad-buildd:variant-binarypackage-builds into launchpad-buildd:master
Michael Hudson-Doyle has proposed merging ~mwhudson/launchpad-buildd:variant-binarypackage-builds into launchpad-buildd:master with ~mwhudson/launchpad-buildd:separate-isa-abi-tags as a prerequisite.
Commit message:
Support architecture variants for binarypackage builds.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~mwhudson/launchpad-buildd/+git/launchpad-buildd/+merge/490635
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~mwhudson/launchpad-buildd:variant-binarypackage-builds into launchpad-buildd:master.
diff --git a/lpbuildd/binarypackage.py b/lpbuildd/binarypackage.py
index 5b7bc54..0495279 100644
--- a/lpbuildd/binarypackage.py
+++ b/lpbuildd/binarypackage.py
@@ -68,14 +68,18 @@ class DpkgArchitectureCache:
def __init__(self):
self._matches = {}
- def match(self, arch, wildcard):
- if (arch, wildcard) not in self._matches:
- command = ["dpkg-architecture", "-a%s" % arch, "-i%s" % wildcard]
+ def match(self, abi_tag, wildcard):
+ if (abi_tag, wildcard) not in self._matches:
+ command = [
+ "dpkg-architecture",
+ "-a%s" % abi_tag,
+ "-i%s" % wildcard,
+ ]
env = dict(os.environ)
env.pop("DEB_HOST_ARCH", None)
ret = subprocess.call(command, env=env) == 0
- self._matches[(arch, wildcard)] = ret
- return self._matches[(arch, wildcard)]
+ self._matches[(abi_tag, wildcard)] = ret
+ return self._matches[(abi_tag, wildcard)]
dpkg_architecture = DpkgArchitectureCache()
@@ -178,10 +182,10 @@ class BinaryPackageBuildManager(DebianBuildManager):
currently_building.write(currently_building_contents)
os.fchmod(currently_building.fileno(), 0o644)
- args = ["sbuild-package", self._buildid, self.arch_tag]
+ args = ["sbuild-package", self._buildid, self.abi_tag]
args.append(self.suite)
args.extend(["-c", "chroot:build-%s" % self._buildid])
- args.append("--arch=" + self.arch_tag)
+ args.append("--arch=" + self.abi_tag)
args.append("--dist=" + self.suite)
args.append("--nolog")
if self.arch_indep:
@@ -192,6 +196,10 @@ class BinaryPackageBuildManager(DebianBuildManager):
env.pop("DEB_BUILD_OPTIONS", None)
else:
env["DEB_BUILD_OPTIONS"] = "noautodbgsym"
+ if self.isa_tag != self.abi_tag:
+ env["DEB_HOST_ARCH_VARIANT"] = self.isa_tag
+ suffix = self.isa_tag[len(self.abi_tag) :]
+ args.append("--dpkg-file-suffix=" + suffix)
self.runSubProcess(self._sbuildpath, args, env=env)
def getAptLists(self):
@@ -305,7 +313,7 @@ class BinaryPackageBuildManager(DebianBuildManager):
if dep_arch is not None:
arch_match = False
for enabled, arch_wildcard in dep_arch:
- if dpkg_architecture.match(self.arch_tag, arch_wildcard):
+ if dpkg_architecture.match(self.abi_tag, arch_wildcard):
arch_match = enabled
break
elif not enabled:
diff --git a/lpbuildd/builder.py b/lpbuildd/builder.py
index 7ffa3be..857d70c 100644
--- a/lpbuildd/builder.py
+++ b/lpbuildd/builder.py
@@ -193,8 +193,8 @@ class BuildManager:
command,
"--backend=%s" % self.backend_name,
"--series=%s" % self.series,
- "--abi-tag=%s" % self.arch_tag,
- "--isa-tag=%s" % self.arch_tag,
+ "--abi-tag=%s" % self.abi_tag,
+ "--isa-tag=%s" % self.isa_tag,
]
for constraint in self.constraints:
base_args.append("--constraint=%s" % constraint)
@@ -279,7 +279,13 @@ class BuildManager:
self.image_type = extra_args.get("image_type", "chroot")
self.series = extra_args["series"]
- self.arch_tag = extra_args.get("arch_tag", self._builder.getArch())
+ if "abi_tag" in extra_args and "isa_tag" in extra_args:
+ self.abi_tag = extra_args["abi_tag"]
+ self.isa_tag = extra_args["isa_tag"]
+ else:
+ self.abi_tag = self.isa_tag = extra_args.get(
+ "arch_tag", self._builder.getArch()
+ )
self.fast_cleanup = extra_args.get("fast_cleanup", False)
self.constraints = extra_args.get("builder_constraints") or []
@@ -294,7 +300,8 @@ class BuildManager:
self.backend_name,
self._buildid,
series=self.series,
- arch=self.arch_tag,
+ abi_tag=self.abi_tag,
+ isa_tag=self.isa_tag,
constraints=self.constraints,
)
diff --git a/lpbuildd/debian.py b/lpbuildd/debian.py
index 5e52884..5be4fa6 100644
--- a/lpbuildd/debian.py
+++ b/lpbuildd/debian.py
@@ -102,7 +102,7 @@ class DebianBuildManager(BuildManager):
yield filename
def getChangesFilename(self):
- changes = self._dscfile[:-4] + "_" + self.arch_tag + ".changes"
+ changes = self._dscfile[:-4] + "_" + self.isa_tag + ".changes"
return get_build_path(self.home, self._buildid, changes)
def gatherResults(self):
diff --git a/lpbuildd/target/backend.py b/lpbuildd/target/backend.py
index 371af8f..1d6222f 100644
--- a/lpbuildd/target/backend.py
+++ b/lpbuildd/target/backend.py
@@ -32,10 +32,18 @@ class Backend:
supports_snapd = False
- def __init__(self, build_id, series=None, arch=None, constraints=None):
+ def __init__(
+ self,
+ build_id,
+ series=None,
+ abi_tag=None,
+ isa_tag=None,
+ constraints=None,
+ ):
self.build_id = build_id
self.series = series
- self.arch = arch
+ self.abi_tag = abi_tag
+ self.isa_tag = isa_tag
self.constraints = constraints or []
self.build_path = os.path.join(os.environ["HOME"], "build-" + build_id)
@@ -228,7 +236,9 @@ class Backend:
rmtree(tmp_dir)
-def make_backend(name, build_id, series=None, arch=None, constraints=None):
+def make_backend(
+ name, build_id, series=None, abi_tag=None, isa_tag=None, constraints=None
+):
if name == "chroot":
from lpbuildd.target.chroot import Chroot
@@ -250,5 +260,9 @@ def make_backend(name, build_id, series=None, arch=None, constraints=None):
else:
raise KeyError("Unknown backend: %s" % name)
return backend_factory(
- build_id, series=series, arch=arch, constraints=constraints
+ build_id,
+ series=series,
+ abi_tag=abi_tag,
+ isa_tag=isa_tag,
+ constraints=constraints,
)
diff --git a/lpbuildd/target/chroot.py b/lpbuildd/target/chroot.py
index d690ec1..0eca4f0 100644
--- a/lpbuildd/target/chroot.py
+++ b/lpbuildd/target/chroot.py
@@ -63,8 +63,8 @@ class Chroot(Backend):
+ [f"{key}={value}" for key, value in env.items()]
+ args
)
- if self.arch is not None:
- args = set_personality(args, self.arch, series=self.series)
+ if self.abi_tag is not None:
+ args = set_personality(args, self.abi_tag, series=self.series)
if cwd is not None:
# This requires either a helper program in the chroot or
# unpleasant quoting. For now we go for the unpleasant quoting,
diff --git a/lpbuildd/target/lxd.py b/lpbuildd/target/lxd.py
index ab35590..dc3bafb 100644
--- a/lpbuildd/target/lxd.py
+++ b/lpbuildd/target/lxd.py
@@ -117,11 +117,11 @@ class LXD(Backend):
@property
def lxc_arch(self):
- return self.arches[self.arch]
+ return self.arches[self.abi_tag]
@property
def alias(self):
- return f"lp-{self.series}-{self.arch}"
+ return f"lp-{self.series}-{self.abi_tag}"
@property
def name(self):
@@ -142,9 +142,9 @@ class LXD(Backend):
"properties": {
"os": "Ubuntu",
"series": self.series,
- "architecture": self.arch,
+ "architecture": self.abi_tag,
"description": (
- f"Launchpad chroot for Ubuntu {self.series} ({self.arch})"
+ f"Launchpad chroot for Ubuntu {self.series} ({self.abi_tag})"
),
},
}
@@ -476,7 +476,7 @@ class LXD(Backend):
# Linux 4.4 on powerpc doesn't support all the seccomp bits that LXD
# needs.
- if self.arch == "powerpc":
+ if self.abi_tag == "powerpc":
raw_lxc_config.append(("lxc.seccomp", ""))
config = {
"security.privileged": "true",
@@ -682,7 +682,7 @@ class LXD(Backend):
]
)
- if self.arch == "armhf":
+ if self.abi_tag == "armhf":
# Work around https://github.com/lxc/lxcfs/issues/553. In
# principle that could result in over-reporting the number of
# available CPU cores, but that isn't a concern in
@@ -731,8 +731,8 @@ class LXD(Backend):
if env:
for key, value in env.items():
env_params.extend(["--env", f"{key}={value}"])
- if self.arch is not None:
- args = set_personality(args, self.arch, series=self.series)
+ if self.abi_tag is not None:
+ args = set_personality(args, self.abi_tag, series=self.series)
if cwd is not None:
# This requires either a helper program in the chroot or
# unpleasant quoting. For now we go for the unpleasant quoting,
diff --git a/lpbuildd/target/operation.py b/lpbuildd/target/operation.py
index 1fd248c..1c7b7f3 100644
--- a/lpbuildd/target/operation.py
+++ b/lpbuildd/target/operation.py
@@ -30,7 +30,7 @@ class Operation:
parser.add_argument(
"--isa-tag",
metavar="ISA",
- help="(currently ununsed)",
+ help="build for ISA",
)
parser.add_argument(
"--constraint",
@@ -49,7 +49,8 @@ class Operation:
self.args.backend,
self.args.build_id,
series=self.args.series,
- arch=self.args.abi_tag,
+ abi_tag=self.args.abi_tag,
+ isa_tag=self.args.isa_tag,
constraints=self.args.constraints,
)
diff --git a/lpbuildd/tests/fakebuilder.py b/lpbuildd/tests/fakebuilder.py
index 15a820a..c3bafe1 100644
--- a/lpbuildd/tests/fakebuilder.py
+++ b/lpbuildd/tests/fakebuilder.py
@@ -259,8 +259,8 @@ class UncontainedBackend(Backend):
]
+ args
)
- if self.arch is not None:
- args = set_personality(args, self.arch, series=self.series)
+ if self.abi_tag is not None:
+ args = set_personality(args, self.abi_tag, series=self.series)
if input_text is None and not get_output:
subprocess.check_call(args, cwd=cwd, **kwargs)
else:
diff --git a/lpbuildd/tests/test_binarypackage.py b/lpbuildd/tests/test_binarypackage.py
index 3e007fc..51d007c 100644
--- a/lpbuildd/tests/test_binarypackage.py
+++ b/lpbuildd/tests/test_binarypackage.py
@@ -19,6 +19,7 @@ from testtools.matchers import (
Is,
MatchesListwise,
Not,
+ StartsWith,
)
from testtools.twistedsupport import AsynchronousDeferredRunTest
from twisted.internet import defer
@@ -112,13 +113,23 @@ class TestBinaryPackageBuildManagerIteration(TestCase):
return self.buildmanager._state
@defer.inlineCallbacks
- def startBuild(self, dscname=""):
+ def startBuild(self, dscname="", abi_tag=None, isa_tag=None):
# The build manager's iterate() kicks off the consecutive states
# after INIT.
+ extra_args = {
+ "series": "warty",
+ "suite": "warty",
+ "ogrecomponent": "main",
+ }
+ if abi_tag is not None:
+ extra_args["abi_tag"] = abi_tag
+ if isa_tag is not None:
+ extra_args["isa_tag"] = isa_tag
+
self.buildmanager.initiate(
{"foo_1.dsc": dscname},
"chroot.tar.gz",
- {"series": "warty", "suite": "warty", "ogrecomponent": "main"},
+ extra_args,
)
os.makedirs(self.chrootdir)
@@ -127,30 +138,51 @@ class TestBinaryPackageBuildManagerIteration(TestCase):
# SBUILD.
self.buildmanager._state = BinaryPackageBuildState.UPDATE
+ sbuild_args = [
+ "sharepath/bin/sbuild-package",
+ "sbuild-package",
+ self.buildid,
+ "i386",
+ "warty",
+ "-c",
+ "chroot:build-" + self.buildid,
+ "--arch=i386",
+ "--dist=warty",
+ "--nolog",
+ "foo_1.dsc",
+ ]
+
+ if abi_tag != isa_tag:
+ env_matcher = ContainsDict(
+ {"DEB_HOST_ARCH_VARIANT": Equals(isa_tag)}
+ )
+ suffix = isa_tag[len(abi_tag) :]
+ sbuild_args.append("--dpkg-file-suffix=" + suffix)
+ else:
+ env_matcher = Not(Contains("DEB_HOST_ARCH_VARIANT"))
+
# SBUILD: Build the package.
yield self.buildmanager.iterate(0)
self.assertState(
BinaryPackageBuildState.SBUILD,
- [
- "sharepath/bin/sbuild-package",
- "sbuild-package",
- self.buildid,
- "i386",
- "warty",
- "-c",
- "chroot:build-" + self.buildid,
- "--arch=i386",
- "--dist=warty",
- "--nolog",
- "foo_1.dsc",
- ],
+ sbuild_args,
+ env_matcher=env_matcher,
final=True,
)
self.assertFalse(self.builder.wasCalled("chrootFail"))
def assertState(self, state, command, env_matcher=None, final=False):
self.assertEqual(state, self.getState())
- self.assertEqual(command, self.buildmanager.commands[-1][0])
+ matchers = []
+ for arg in command:
+ if isinstance(arg, str):
+ matchers.append(Equals(arg))
+ else:
+ matchers.append(arg)
+ self.assertThat(
+ self.buildmanager.commands[-1][0],
+ MatchesListwise(matchers),
+ )
if env_matcher is not None:
self.assertThat(self.buildmanager.commands[-1][1], env_matcher)
if final:
@@ -174,8 +206,8 @@ class TestBinaryPackageBuildManagerIteration(TestCase):
"scan-for-processes",
"--backend=chroot",
"--series=warty",
- "--abi-tag=i386",
- "--isa-tag=i386",
+ StartsWith("--abi-tag="),
+ StartsWith("--isa-tag="),
self.buildid,
],
final=False,
@@ -191,8 +223,8 @@ class TestBinaryPackageBuildManagerIteration(TestCase):
"umount-chroot",
"--backend=chroot",
"--series=warty",
- "--abi-tag=i386",
- "--isa-tag=i386",
+ StartsWith("--abi-tag="),
+ StartsWith("--isa-tag="),
self.buildid,
],
final=True,
@@ -231,6 +263,38 @@ class TestBinaryPackageBuildManagerIteration(TestCase):
self.assertFalse(self.builder.wasCalled("buildFail"))
@defer.inlineCallbacks
+ def test_variant(self):
+ # The build manager iterates a normal build from start to finish.
+ yield self.startBuild(abi_tag="i386", isa_tag="i386sse")
+
+ write_file(
+ os.path.join(self.buildmanager._cachepath, "buildlog"),
+ "I am a build log.",
+ )
+ changes_path = os.path.join(
+ self.buildmanager.home,
+ "build-%s" % self.buildid,
+ "foo_1_i386sse.changes",
+ )
+ write_file(changes_path, "I am a changes file.")
+
+ # After building the package, reap processes.
+ yield self.assertScansSanely(SBuildExitCodes.OK)
+ self.assertFalse(self.builder.wasCalled("buildFail"))
+ self.assertThat(
+ self.builder,
+ HasWaitingFiles.byEquality(
+ {
+ "foo_1_i386sse.changes": b"I am a changes file.",
+ }
+ ),
+ )
+
+ # Control returns to the DebianBuildManager in the UMOUNT state.
+ self.assertUnmountsSanely()
+ self.assertFalse(self.builder.wasCalled("buildFail"))
+
+ @defer.inlineCallbacks
def test_with_debug_symbols(self):
# A build with debug symbols sets up /CurrentlyBuilding
# appropriately, and does not pass DEB_BUILD_OPTIONS.
diff --git a/lpbuildd/util.py b/lpbuildd/util.py
index 9231206..bd949ad 100644
--- a/lpbuildd/util.py
+++ b/lpbuildd/util.py
@@ -22,8 +22,8 @@ def shell_escape(s):
return quote(s)
-def get_arch_bits(arch):
- if arch == "x32":
+def get_arch_bits(abi_tag):
+ if abi_tag == "x32":
# x32 is an exception: the userspace is 32-bit, but it expects to be
# running on a 64-bit kernel.
return 64
@@ -31,7 +31,7 @@ def get_arch_bits(arch):
env = dict(os.environ)
env.pop("DEB_HOST_ARCH_BITS", None)
bits = subprocess.check_output(
- ["dpkg-architecture", "-a%s" % arch, "-qDEB_HOST_ARCH_BITS"],
+ ["dpkg-architecture", "-a%s" % abi_tag, "-qDEB_HOST_ARCH_BITS"],
env=env,
universal_newlines=True,
).rstrip("\n")
@@ -42,12 +42,12 @@ def get_arch_bits(arch):
else:
raise RuntimeError(
"Don't know how to deal with architecture %s "
- "(DEB_HOST_ARCH_BITS=%s)" % (arch, bits)
+ "(DEB_HOST_ARCH_BITS=%s)" % (abi_tag, bits)
)
-def set_personality(args, arch, series=None):
- bits = get_arch_bits(arch)
+def set_personality(args, abi_tag, series=None):
+ bits = get_arch_bits(abi_tag)
assert bits in (32, 64)
if bits == 32:
setarch_cmd = ["linux32"]
diff --git a/sbuildrc b/sbuildrc
index 008f034..7da1464 100644
--- a/sbuildrc
+++ b/sbuildrc
@@ -30,8 +30,10 @@ $build_environment = {
# We want to expose almost nothing from the buildd environment.
# DEB_BUILD_OPTIONS is set by sbuild-package.
+# DEB_HOST_ARCH_VARIANT is set by binarypackage.py.
$environment_filter = [
'^DEB_BUILD_OPTIONS$',
+ '^DEB_HOST_ARCH_VARIANT$',
];
# We're just going to throw the chroot away anyway.
