launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #32973
[Merge] ~artemstreltsov/launchpad-buildd:add_docker26_support into launchpad-buildd:master
Artem Streltsov has proposed merging ~artemstreltsov/launchpad-buildd:add_docker26_support into launchpad-buildd:master.
Commit message:
Add docker v26 support
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~artemstreltsov/launchpad-buildd/+git/launchpad-buildd/+merge/492535
Fixes build failures on Docker 26.x where docker saves images as an OCI image layout. Also, unpins the ppa with an older docker version, which was a temporary fix.
Converts the new layout into (which is what the publisher expects):
- manifest.json
- config.json
- <layer-digest>.tar.gz
- digests.json
How to verify:
- do a local oci build
- check artifacts
The previous MP created digests.json incorrectly. Now I adapt docker26 manifest.json and reuse the rest of the logic.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~artemstreltsov/launchpad-buildd:add_docker26_support into launchpad-buildd:master.
diff --git a/lpbuildd/oci.py b/lpbuildd/oci.py
index 5ce2cd8..3d58af7 100644
--- a/lpbuildd/oci.py
+++ b/lpbuildd/oci.py
@@ -163,6 +163,7 @@ class OCIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
current_dir = ""
gzip_layer = None
symlinks = []
+ fileobj = None
try:
# The tarfile is a stream and must be processed in order
for file in tar:
@@ -185,6 +186,8 @@ class OCIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
)
symlinks.append(file)
continue
+ if file.name.startswith("blobs/"):
+ tar.extract(file, extract_path)
if current_dir and file.name.endswith("layer.tar"):
# This is the actual layer data.
# Instead of adding the layer.tar to a gzip directory
@@ -215,7 +218,8 @@ class OCIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
finally:
if gzip_layer is not None:
gzip_layer.close()
- fileobj.close()
+ if fileobj is not None:
+ fileobj.close()
# deal with any symlinks we had
for symlink in symlinks:
@@ -234,6 +238,10 @@ class OCIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
)
shutil.copy(source_name, target_name)
+ oci_layout_path = os.path.join(extract_path, "oci-layout")
+ if os.path.exists(oci_layout_path):
+ self._normalizeImageLayout(extract_path)
+
# We need these mapping files
sha_directory = tempfile.mkdtemp()
# This can change depending on the kernel options / docker package
@@ -281,3 +289,40 @@ class OCIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
except Exception as e:
self._builder.log(f"Failed to parse manifest: {e}")
raise
+
+ def _normalizeImageLayout(self, extract_path):
+ manifest_path = os.path.join(extract_path, "manifest.json")
+ with open(manifest_path) as fp:
+ manifest = json.load(fp)
+
+ config_rel_path = manifest[0]["Config"]
+ config_path = os.path.join(extract_path, config_rel_path)
+ config_hex = os.path.basename(config_rel_path)
+ config_out_path = os.path.join(extract_path, f"{config_hex}.json")
+ shutil.copy(config_path, config_out_path)
+
+ repo_tags = manifest[0]["RepoTags"]
+
+ layers = manifest[0]["Layers"]
+ layer_hexes = []
+
+ for layer_path in layers:
+ # layer_path is e.g. "blobs/sha256/<hex>"
+ hex_id = os.path.basename(layer_path)
+ src = os.path.join(extract_path, layer_path)
+ out_path = os.path.join(extract_path, f"{hex_id}.tar.gz")
+
+ with open(src, "rb") as inf, gzip.open(out_path, "wb") as outf:
+ shutil.copyfileobj(inf, outf)
+
+ layer_hexes.append(hex_id)
+
+ manifest = [
+ {
+ "Config": f"{config_hex}.json",
+ "RepoTags": repo_tags,
+ "Layers": [f"{hex_id}/layer.tar" for hex_id in layer_hexes],
+ }
+ ]
+ with open(manifest_path, "w") as fp:
+ json.dump(manifest, fp)
diff --git a/lpbuildd/target/build_oci.py b/lpbuildd/target/build_oci.py
index 22d30df..28c4869 100644
--- a/lpbuildd/target/build_oci.py
+++ b/lpbuildd/target/build_oci.py
@@ -79,25 +79,7 @@ class BuildOCI(
self._add_docker_engine_proxy_settings()
deps.extend(self.vcs_deps)
self.backend.run(["apt-get", "-y", "install"] + deps)
- # XXX jchittum: pin docker.io to last known working version
- # provided by the Ubuntu Server team via a PPA
- # the PPA version contains an epoch, and will sort higher in version
- # to the archive. To revert, simply delete the addition of the PPA
- # The PPA only contains docker.io.
- # For more info: https://bugs.launchpad.net/launchpad/+bug/2098106
- # software-properties-common required for add-apt-repository
- # we do not want to handle the entire process ourselves
- # and assuming a buildd base for the lxd container, it will not
- # have software-properties-common installed by default
- self.backend.run(
- ["apt-get", "-y", "install", "software-properties-common"]
- )
- self.backend.run(
- ["add-apt-repository", "-y", "ppa:canonical-server/lp2098106-docker-rollback"]
- )
- self.backend.run(
- ["apt-get", "-y", "install", "docker.io"]
- )
+ self.backend.run(["apt-get", "-y", "install", "docker.io"])
if self.backend.supports_snapd:
self.snap_store_set_proxy()
self.backend.run(["systemctl", "restart", "docker"])
@@ -120,7 +102,12 @@ class BuildOCI(
logger.info("Running build phase...")
args = ["docker", "build", "--no-cache"]
if self.args.proxy_url:
- for var in ("http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY"):
+ for var in (
+ "http_proxy",
+ "HTTP_PROXY",
+ "https_proxy",
+ "HTTPS_PROXY",
+ ):
args.extend(["--build-arg", f"{var}={self.args.proxy_url}"])
args.extend(["--tag", self.args.name])
if self.args.build_file is not None:
diff --git a/lpbuildd/target/tests/test_build_oci.py b/lpbuildd/target/tests/test_build_oci.py
index 1b47c2d..46a3343 100644
--- a/lpbuildd/target/tests/test_build_oci.py
+++ b/lpbuildd/target/tests/test_build_oci.py
@@ -100,11 +100,6 @@ class TestBuildOCI(TestCase):
MatchesListwise(
[
RanAptGet("install", "bzr"),
- RanAptGet("install", "software-properties-common"),
- RanCommand(
- ["add-apt-repository",
- "-y",
- "ppa:canonical-server/lp2098106-docker-rollback"]),
RanAptGet("install", "docker.io"),
RanCommand(["systemctl", "restart", "docker"]),
RanCommand(["mkdir", "-p", "/home/buildd"]),
@@ -131,11 +126,6 @@ class TestBuildOCI(TestCase):
MatchesListwise(
[
RanAptGet("install", "git"),
- RanAptGet("install", "software-properties-common"),
- RanCommand(
- ["add-apt-repository",
- "-y",
- "ppa:canonical-server/lp2098106-docker-rollback"]),
RanAptGet("install", "docker.io"),
RanCommand(["systemctl", "restart", "docker"]),
RanCommand(["mkdir", "-p", "/home/buildd"]),
@@ -224,11 +214,6 @@ class TestBuildOCI(TestCase):
["mkdir", "-p", "/etc/systemd/system/docker.service.d"]
),
RanAptGet("install", "python3", "socat", "git"),
- RanAptGet("install", "software-properties-common"),
- RanCommand(
- ["add-apt-repository",
- "-y",
- "ppa:canonical-server/lp2098106-docker-rollback"]),
RanAptGet("install", "docker.io"),
RanCommand(["systemctl", "restart", "docker"]),
RanCommand(["mkdir", "-p", "/home/buildd"]),
@@ -753,11 +738,6 @@ class TestBuildOCI(TestCase):
build_oci.backend.run.calls,
MatchesAll(
AnyMatch(RanAptGet("install", "bzr")),
- AnyMatch(RanAptGet("install", "software-properties-common")),
- AnyMatch(RanCommand(
- ["add-apt-repository",
- "-y",
- "ppa:canonical-server/lp2098106-docker-rollback"])),
AnyMatch(RanAptGet("install", "docker.io")),
AnyMatch(
RanBuildCommand(
