launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #27172
[Merge] ~cjwatson/launchpad:oci-arch-compat into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:oci-arch-compat into launchpad:master.
Commit message:
Fix platform specifiers in OCI multi-arch manifests
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
Related bugs:
Bug #1930094 in Launchpad itself: "OCI: LP should use "ppc64le" for image manifests"
https://bugs.launchpad.net/launchpad/+bug/1930094
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/403552
The platform specifier isn't always as simple as `{"os": "linux", "architecture": dpkg_architecture_name}`, because the manifest list specification defers to Go for its architecture names, not dpkg; and there's additional variant information that we should send for ARM architectures. See:
https://docs.docker.com/registry/spec/manifest-v2-2/#manifest-list-field-descriptions
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:oci-arch-compat into launchpad:master.
diff --git a/lib/lp/oci/model/ociregistryclient.py b/lib/lp/oci/model/ociregistryclient.py
index f3e623b..dbed50c 100644
--- a/lib/lp/oci/model/ociregistryclient.py
+++ b/lib/lp/oci/model/ociregistryclient.py
@@ -404,20 +404,45 @@ class OCIRegistryClient:
# Avoid traceback reference cycles.
del exceptions
+ _arch_to_platform = {
+ "amd64": {"os": "linux", "architecture": "amd64"},
+ "arm64": {"os": "linux", "architecture": "arm64", "variant": "v8"},
+ "armhf": {"os": "linux", "architecture": "arm", "variant": "v7"},
+ "i386": {"os": "linux", "architecture": "386"},
+ "ppc64el": {"os": "linux", "architecture": "ppc64le"},
+ "riscv64": {"os": "linux", "architecture": "riscv64"},
+ "s390x": {"os": "linux", "architecture": "s390x"},
+ }
+
+ @classmethod
+ def _makePlatformSpecifiers(cls, arch):
+ """Make OCI platform specifiers for an architecture tag.
+
+ OCI platform specifiers are based on Go architecture names, which
+ aren't identical to dpkg architecture tags in all cases. Do the
+ necessary mapping.
+
+ The first specifier in the list returned by this method is the
+ preferred one; subsequent specifiers may have been set by old code
+ and should still be searched for when finding existing manifests.
+ """
+ platforms = []
+ oci_platform = cls._arch_to_platform.get(arch)
+ if oci_platform is not None:
+ platforms.append(oci_platform)
+ else:
+ log.warning("No OCI platform specifier known for %s", arch)
+ legacy_platform = {"os": "linux", "architecture": arch}
+ if legacy_platform != oci_platform:
+ platforms.append(legacy_platform)
+ return platforms
+
@classmethod
def makeMultiArchManifest(cls, http_client, push_rule, build_request,
uploaded_builds, tag):
"""Returns the multi-arch manifest content including all uploaded
builds of the given build_request.
"""
- def get_manifest_for_architecture(manifests, arch):
- """Find, in the manifests list, the manifest for the given arch."""
- expected_platform = {"architecture": arch, "os": "linux"}
- for m in manifests:
- if m["platform"] == expected_platform:
- return m
- return None
-
try:
current_manifest = cls._getCurrentRegistryManifest(
http_client, tag)
@@ -454,9 +479,11 @@ class OCIRegistryClient:
log.info("Build manifest found for build {}".format(build.id))
digest = build_manifest["digest"]
size = build_manifest["size"]
- arch = build.processor.name
+ arch = build.distro_arch_series.architecturetag
+ platforms = cls._makePlatformSpecifiers(arch)
- manifest = get_manifest_for_architecture(manifests, arch)
+ manifest = next(
+ (m for m in manifests if m["platform"] in platforms), None)
if manifest is None:
log.info(
"Appending multi-arch manifest for build {} "
@@ -466,7 +493,7 @@ class OCIRegistryClient:
"vnd.docker.distribution.manifest.v2+json"),
"size": size,
"digest": digest,
- "platform": {"architecture": arch, "os": "linux"}
+ "platform": platforms[0],
}
manifests.append(manifest)
else:
@@ -475,7 +502,7 @@ class OCIRegistryClient:
"with arch {}".format(build.id, arch))
manifest["digest"] = digest
manifest["size"] = size
- manifest["platform"]["architecture"] = arch
+ manifest["platform"] = platforms[0]
return current_manifest
diff --git a/lib/lp/oci/tests/test_ociregistryclient.py b/lib/lp/oci/tests/test_ociregistryclient.py
index 44241bc..2fa179e 100644
--- a/lib/lp/oci/tests/test_ociregistryclient.py
+++ b/lib/lp/oci/tests/test_ociregistryclient.py
@@ -125,8 +125,9 @@ class TestOCIRegistryClient(OCIConfigHelperMixin, SpyProxyCallsMixin,
# This produces a git ref that does not match the 'valid' OCI branch
# format, so will not get multiple tags. Multiple tags are tested
# explicitly.
- [git_ref] = self.factory.makeGitRefs(paths=['refs/heads/v1.0-20.04'])
- recipe = self.factory.makeOCIRecipe(git_ref=git_ref)
+ [self.git_ref] = self.factory.makeGitRefs(
+ paths=['refs/heads/v1.0-20.04'])
+ recipe = self.factory.makeOCIRecipe(git_ref=self.git_ref)
self.build = self.factory.makeOCIRecipeBuild(recipe=recipe)
self.push_rule = self.factory.makeOCIPushRule(recipe=self.build.recipe)
self.client = OCIRegistryClient()
@@ -687,6 +688,33 @@ class TestOCIRegistryClient(OCIConfigHelperMixin, SpyProxyCallsMixin,
}),
))
+ def test_platform_specifiers(self):
+ expected_platforms = {
+ "amd64": [{"os": "linux", "architecture": "amd64"}],
+ "arm64": [
+ {"os": "linux", "architecture": "arm64", "variant": "v8"},
+ {"os": "linux", "architecture": "arm64"},
+ ],
+ "armhf": [
+ {"os": "linux", "architecture": "arm", "variant": "v7"},
+ {"os": "linux", "architecture": "armhf"},
+ ],
+ "i386": [
+ {"os": "linux", "architecture": "386"},
+ {"os": "linux", "architecture": "i386"},
+ ],
+ "ppc64el": [
+ {"os": "linux", "architecture": "ppc64le"},
+ {"os": "linux", "architecture": "ppc64el"},
+ ],
+ "riscv64": [{"os": "linux", "architecture": "riscv64"}],
+ "s390x": [{"os": "linux", "architecture": "s390x"}],
+ "unknown": [{"os": "linux", "architecture": "unknown"}],
+ }
+ for arch, platforms in expected_platforms.items():
+ self.assertEqual(
+ platforms, self.client._makePlatformSpecifiers(arch))
+
@responses.activate
def test_multi_arch_manifest_upload_skips_superseded_builds(self):
recipe = self.build.recipe
@@ -718,18 +746,23 @@ class TestOCIRegistryClient(OCIConfigHelperMixin, SpyProxyCallsMixin,
"""Ensure that multi-arch manifest upload works and tags correctly
the uploaded image."""
# Creates a build request with 2 builds.
- recipe = self.build.recipe
- build1 = self.build
+ recipe = self.factory.makeOCIRecipe(git_ref=self.git_ref)
+ distroseries = self.factory.makeDistroSeries(
+ distribution=recipe.distribution, status=SeriesStatus.CURRENT)
+ for architecturetag, processor_name in (
+ ("i386", "386"),
+ ("amd64", "amd64"),
+ ("hppa", "hppa"),
+ ):
+ self.factory.makeDistroArchSeries(
+ distroseries=distroseries, architecturetag=architecturetag,
+ processor=getUtility(IProcessorSet).getByName(processor_name))
+ build1 = self.factory.makeOCIRecipeBuild(
+ recipe=recipe, distro_arch_series=distroseries["i386"])
build2 = self.factory.makeOCIRecipeBuild(
- recipe=recipe)
+ recipe=recipe, distro_arch_series=distroseries["amd64"])
build3 = self.factory.makeOCIRecipeBuild(
- recipe=recipe)
- naked_build1 = removeSecurityProxy(build1)
- naked_build2 = removeSecurityProxy(build2)
- naked_build3 = removeSecurityProxy(build3)
- naked_build1.processor = getUtility(IProcessorSet).getByName('386')
- naked_build2.processor = getUtility(IProcessorSet).getByName('amd64')
- naked_build3.processor = getUtility(IProcessorSet).getByName('hppa')
+ recipe=recipe, distro_arch_series=distroseries["hppa"])
# Creates a mock IOCIRecipeRequestBuildsJobSource, as it was created
# by the celery job and triggered the 3 registry uploads already.
@@ -746,7 +779,7 @@ class TestOCIRegistryClient(OCIConfigHelperMixin, SpyProxyCallsMixin,
ZopeUtilityFixture(job_source, IOCIRecipeRequestBuildsJobSource))
build_request = OCIRecipeBuildRequest(recipe, -1)
- push_rule = self.build.recipe.push_rules[0]
+ push_rule = self.factory.makeOCIPushRule(recipe=recipe)
responses.add(
"GET", "{}/v2/{}/manifests/v1.0-20.04_edge".format(
push_rule.registry_url, push_rule.image_name),
@@ -809,14 +842,20 @@ class TestOCIRegistryClient(OCIConfigHelperMixin, SpyProxyCallsMixin,
# Creates a build request with 2 builds: amd64 (which is already in
# the manifest) and hppa (that should be added)
- recipe = self.build.recipe
- build1 = self.build
+ recipe = self.factory.makeOCIRecipe(git_ref=self.git_ref)
+ distroseries = self.factory.makeDistroSeries(
+ distribution=recipe.distribution, status=SeriesStatus.CURRENT)
+ for architecturetag, processor_name in (
+ ("amd64", "amd64"),
+ ("hppa", "hppa"),
+ ):
+ self.factory.makeDistroArchSeries(
+ distroseries=distroseries, architecturetag=architecturetag,
+ processor=getUtility(IProcessorSet).getByName(processor_name))
+ build1 = self.factory.makeOCIRecipeBuild(
+ recipe=recipe, distro_arch_series=distroseries["amd64"])
build2 = self.factory.makeOCIRecipeBuild(
- recipe=recipe)
- naked_build1 = removeSecurityProxy(build1)
- naked_build2 = removeSecurityProxy(build2)
- naked_build1.processor = getUtility(IProcessorSet).getByName('amd64')
- naked_build2.processor = getUtility(IProcessorSet).getByName('hppa')
+ recipe=recipe, distro_arch_series=distroseries["hppa"])
# Creates a mock IOCIRecipeRequestBuildsJobSource, as it was created
# by the celery job and triggered the 3 registry uploads already.
@@ -832,7 +871,7 @@ class TestOCIRegistryClient(OCIConfigHelperMixin, SpyProxyCallsMixin,
ZopeUtilityFixture(job_source, IOCIRecipeRequestBuildsJobSource))
build_request = OCIRecipeBuildRequest(recipe, -1)
- push_rule = self.build.recipe.push_rules[0]
+ push_rule = self.factory.makeOCIPushRule(recipe=recipe)
responses.add(
"GET", "{}/v2/{}/manifests/v1.0-20.04_edge".format(
push_rule.registry_url, push_rule.image_name),
@@ -883,10 +922,14 @@ class TestOCIRegistryClient(OCIConfigHelperMixin, SpyProxyCallsMixin,
current_manifest = {'schemaVersion': 1, 'layers': []}
# Creates a build request with 1 build for amd64.
- recipe = self.build.recipe
- build1 = self.build
- naked_build1 = removeSecurityProxy(build1)
- naked_build1.processor = getUtility(IProcessorSet).getByName('amd64')
+ recipe = self.factory.makeOCIRecipe(git_ref=self.git_ref)
+ distroseries = self.factory.makeDistroSeries(
+ distribution=recipe.distribution, status=SeriesStatus.CURRENT)
+ das = self.factory.makeDistroArchSeries(
+ distroseries=distroseries, architecturetag="amd64",
+ processor=getUtility(IProcessorSet).getByName("amd64"))
+ build1 = self.factory.makeOCIRecipeBuild(
+ recipe=recipe, distro_arch_series=das)
job = mock.Mock()
job.builds = [build1]
@@ -899,7 +942,7 @@ class TestOCIRegistryClient(OCIConfigHelperMixin, SpyProxyCallsMixin,
ZopeUtilityFixture(job_source, IOCIRecipeRequestBuildsJobSource))
build_request = OCIRecipeBuildRequest(recipe, -1)
- push_rule = self.build.recipe.push_rules[0]
+ push_rule = self.factory.makeOCIPushRule(recipe=recipe)
responses.add(
"GET", "{}/v2/{}/manifests/v1.0-20.04_edge".format(
push_rule.registry_url, push_rule.image_name),
