launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #32952
[Merge] ~artemstreltsov/launchpad-buildd:add_docker26.x_support into launchpad-buildd:master
Artem Streltsov has proposed merging ~artemstreltsov/launchpad-buildd:add_docker26.x_support into launchpad-buildd:master.
Commit message:
Add docker 26.x support
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~artemstreltsov/launchpad-buildd/+git/launchpad-buildd/+merge/492120
Fixes build failures on Docker 26.x where docker saves images as an OCI image layout. Also, unpins the ppa with an older docker version, which was a temporary fix.
Converts the new layout into (which is what the publisher expects):
- manifest.json
- config.json
- <layer-digest>.tar.gz
- digests.json
How to verify:
- do a local oci build
- check artifacts
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~artemstreltsov/launchpad-buildd:add_docker26.x_support into launchpad-buildd:master.
diff --git a/lpbuildd/oci.py b/lpbuildd/oci.py
index 5ce2cd8..ffaafd0 100644
--- a/lpbuildd/oci.py
+++ b/lpbuildd/oci.py
@@ -163,6 +163,8 @@ class OCIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
current_dir = ""
gzip_layer = None
symlinks = []
+ oci_layout_detected = False
+ fileobj = None
try:
# The tarfile is a stream and must be processed in order
for file in tar:
@@ -174,6 +176,13 @@ class OCIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
if gzip_layer:
# Close the old directory if we have one
gzip_layer.close()
+ continue
+ # Detect OCI Image Layout and extract everything
+ # for that format
+ if file.name == "oci-layout":
+ tar.extract(file, extract_path)
+ oci_layout_detected = True
+ continue
if file.issym():
# symlinks can't be extracted or derefenced from a stream
# as you can't seek backwards.
@@ -185,6 +194,9 @@ class OCIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
)
symlinks.append(file)
continue
+ if oci_layout_detected:
+ tar.extract(file, extract_path)
+ continue
if current_dir and file.name.endswith("layer.tar"):
# This is the actual layer data.
# Instead of adding the layer.tar to a gzip directory
@@ -195,6 +207,8 @@ class OCIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
# will have to have the name of the directory
# (directory_name.tar.gz/contents) otherwise we will endup
# with multiple gzips with the same name "layer.tar.gz".
+ if fileobj is not None:
+ fileobj.close()
fileobj = tar.extractfile(file)
name = os.path.join(extract_path, f"{current_dir}.tar.gz")
with gzip.GzipFile(name, "wb") as gzip_layer:
@@ -203,9 +217,11 @@ class OCIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
gzip_layer.write(byte)
byte = fileobj.read(1)
elif current_dir and file.name.startswith(current_dir):
- # Other files that are in the layer directories,
- # we don't care about
- continue
+ if current_dir.startswith("blobs"):
+ tar.extract(file, extract_path)
+ else:
+ # Legacy layout: ignore other files
+ continue
else:
# If it's not in a directory, we need that
tar.extract(file, extract_path)
@@ -215,7 +231,8 @@ class OCIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
finally:
if gzip_layer is not None:
gzip_layer.close()
- fileobj.close()
+ if fileobj is not None:
+ fileobj.close()
# deal with any symlinks we had
for symlink in symlinks:
@@ -234,7 +251,12 @@ class OCIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
)
shutil.copy(source_name, target_name)
- # We need these mapping files
+ # If this is an OCI image layout, handle it directly and return.
+ oci_layout_path = os.path.join(extract_path, "oci-layout")
+ if os.path.exists(oci_layout_path):
+ self._gatherOCIResults(extract_path)
+ return
+
sha_directory = tempfile.mkdtemp()
# This can change depending on the kernel options / docker package
# used. This is correct for bionic buildd image
@@ -281,3 +303,72 @@ class OCIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
except Exception as e:
self._builder.log(f"Failed to parse manifest: {e}")
raise
+
+ def _gatherOCIResults(self, extract_path):
+ """Gather results from an OCI Image Layout."""
+ index_path = os.path.join(extract_path, "index.json")
+ with open(index_path) as fp:
+ index = json.load(fp)
+ if not index.get("manifests"):
+ raise RuntimeError("OCI index.json contains no manifests")
+
+ manifest_digest = index["manifests"][0]["digest"]
+ manifest_hex = manifest_digest.split(":", 1)[1]
+ manifest_blob_path = os.path.join(
+ extract_path, "blobs", "sha256", manifest_hex
+ )
+ with open(manifest_blob_path) as fp:
+ manifest = json.load(fp)
+
+ config_digest = manifest["config"]["digest"]
+ config_hex = config_digest.split(":", 1)[1]
+ config_blob_path = os.path.join(
+ extract_path, "blobs", "sha256", config_hex
+ )
+ config_out_path = os.path.join(extract_path, "config.json")
+ shutil.copy(config_blob_path, config_out_path)
+ self._builder.addWaitingFile(config_out_path)
+ with open(config_out_path) as fp:
+ config = json.load(fp)
+
+ layers = manifest.get("layers", [])
+ layer_hexes = []
+ for layer in layers:
+ digest = layer["digest"]
+ hex_id = digest.split(":", 1)[1]
+ src = os.path.join(extract_path, "blobs", "sha256", hex_id)
+ media_type = layer.get("mediaType", "")
+ out_path = os.path.join(extract_path, f"{hex_id}.tar.gz")
+ if media_type.endswith("+gzip"):
+ shutil.copy(src, out_path)
+ else:
+ with open(src, "rb") as inf, gzip.open(out_path, "wb") as outf:
+ shutil.copyfileobj(inf, outf)
+ self._builder.addWaitingFile(out_path)
+ layer_hexes.append(hex_id)
+
+ manifest_path = os.path.join(extract_path, "manifest.json")
+ docker_archive_manifest = [
+ {
+ "Config": "config.json",
+ "Layers": [f"{hex_id}/layer.tar" for hex_id in layer_hexes],
+ }
+ ]
+ with open(manifest_path, "w") as fp:
+ json.dump(docker_archive_manifest, fp)
+ self._builder.addWaitingFile(manifest_path)
+
+ diff_ids = config.get("rootfs", {}).get("diff_ids", [])
+ digest_maps = []
+ mapping = {}
+ for diff_id, hex_id in zip(diff_ids, layer_hexes):
+ mapping[diff_id] = {
+ "digest": hex_id,
+ "source": "",
+ "layer_id": hex_id,
+ }
+ digest_maps.append(mapping)
+ digest_map_file = os.path.join(extract_path, "digests.json")
+ with open(digest_map_file, "w") as fp:
+ json.dump(digest_maps, fp)
+ self._builder.addWaitingFile(digest_map_file)
diff --git a/lpbuildd/target/build_oci.py b/lpbuildd/target/build_oci.py
index 22d30df..28c4869 100644
--- a/lpbuildd/target/build_oci.py
+++ b/lpbuildd/target/build_oci.py
@@ -79,25 +79,7 @@ class BuildOCI(
self._add_docker_engine_proxy_settings()
deps.extend(self.vcs_deps)
self.backend.run(["apt-get", "-y", "install"] + deps)
- # XXX jchittum: pin docker.io to last known working version
- # provided by the Ubuntu Server team via a PPA
- # the PPA version contains an epoch, and will sort higher in version
- # to the archive. To revert, simply delete the addition of the PPA
- # The PPA only contains docker.io.
- # For more info: https://bugs.launchpad.net/launchpad/+bug/2098106
- # software-properties-common required for add-apt-repository
- # we do not want to handle the entire process ourselves
- # and assuming a buildd base for the lxd container, it will not
- # have software-properties-common installed by default
- self.backend.run(
- ["apt-get", "-y", "install", "software-properties-common"]
- )
- self.backend.run(
- ["add-apt-repository", "-y", "ppa:canonical-server/lp2098106-docker-rollback"]
- )
- self.backend.run(
- ["apt-get", "-y", "install", "docker.io"]
- )
+ self.backend.run(["apt-get", "-y", "install", "docker.io"])
if self.backend.supports_snapd:
self.snap_store_set_proxy()
self.backend.run(["systemctl", "restart", "docker"])
@@ -120,7 +102,12 @@ class BuildOCI(
logger.info("Running build phase...")
args = ["docker", "build", "--no-cache"]
if self.args.proxy_url:
- for var in ("http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY"):
+ for var in (
+ "http_proxy",
+ "HTTP_PROXY",
+ "https_proxy",
+ "HTTPS_PROXY",
+ ):
args.extend(["--build-arg", f"{var}={self.args.proxy_url}"])
args.extend(["--tag", self.args.name])
if self.args.build_file is not None:
diff --git a/lpbuildd/target/tests/test_build_oci.py b/lpbuildd/target/tests/test_build_oci.py
index 4ea5132..865443b 100644
--- a/lpbuildd/target/tests/test_build_oci.py
+++ b/lpbuildd/target/tests/test_build_oci.py
@@ -97,11 +97,6 @@ class TestBuildOCI(TestCase):
MatchesListwise(
[
RanAptGet("install", "bzr"),
- RanAptGet("install", "software-properties-common"),
- RanCommand(
- ["add-apt-repository",
- "-y",
- "ppa:canonical-server/lp2098106-docker-rollback"]),
RanAptGet("install", "docker.io"),
RanCommand(["systemctl", "restart", "docker"]),
RanCommand(["mkdir", "-p", "/home/buildd"]),
@@ -127,11 +122,6 @@ class TestBuildOCI(TestCase):
MatchesListwise(
[
RanAptGet("install", "git"),
- RanAptGet("install", "software-properties-common"),
- RanCommand(
- ["add-apt-repository",
- "-y",
- "ppa:canonical-server/lp2098106-docker-rollback"]),
RanAptGet("install", "docker.io"),
RanCommand(["systemctl", "restart", "docker"]),
RanCommand(["mkdir", "-p", "/home/buildd"]),
@@ -218,11 +208,6 @@ class TestBuildOCI(TestCase):
["mkdir", "-p", "/etc/systemd/system/docker.service.d"]
),
RanAptGet("install", "python3", "socat", "git"),
- RanAptGet("install", "software-properties-common"),
- RanCommand(
- ["add-apt-repository",
- "-y",
- "ppa:canonical-server/lp2098106-docker-rollback"]),
RanAptGet("install", "docker.io"),
RanCommand(["systemctl", "restart", "docker"]),
RanCommand(["mkdir", "-p", "/home/buildd"]),
@@ -735,11 +720,6 @@ class TestBuildOCI(TestCase):
build_oci.backend.run.calls,
MatchesAll(
AnyMatch(RanAptGet("install", "bzr")),
- AnyMatch(RanAptGet("install", "software-properties-common")),
- AnyMatch(RanCommand(
- ["add-apt-repository",
- "-y",
- "ppa:canonical-server/lp2098106-docker-rollback"])),
AnyMatch(RanAptGet("install", "docker.io")),
AnyMatch(
RanBuildCommand(
Follow ups
