launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #25457
[Merge] ~pappacena/launchpad:oci-multi-arch-upload into launchpad:master
Thiago F. Pappacena has proposed merging ~pappacena/launchpad:oci-multi-arch-upload into launchpad:master with ~pappacena/launchpad:oci-unify-request-builds as a prerequisite.
Commit message:
Upload multi-arch OCI images
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~pappacena/launchpad/+git/launchpad/+merge/391783
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~pappacena/launchpad:oci-multi-arch-upload into launchpad:master.
diff --git a/database/schema/security.cfg b/database/schema/security.cfg
index 8a9a31b..d5a23f1 100644
--- a/database/schema/security.cfg
+++ b/database/schema/security.cfg
@@ -2710,6 +2710,7 @@ public.account = SELECT
public.archive = SELECT
public.branch = SELECT
public.builder = SELECT
+public.builderprocessor = SELECT
public.buildfarmjob = SELECT, INSERT
public.buildqueue = SELECT, INSERT, UPDATE
public.distribution = SELECT
diff --git a/lib/lp/oci/interfaces/ocirecipe.py b/lib/lp/oci/interfaces/ocirecipe.py
index 0bd3cd2..131c4d3 100644
--- a/lib/lp/oci/interfaces/ocirecipe.py
+++ b/lib/lp/oci/interfaces/ocirecipe.py
@@ -178,6 +178,16 @@ class IOCIRecipeBuildRequest(Interface):
title=_("If set, limit builds to these architecture tags."),
value_type=TextLine(), required=False, readonly=True)
+ uploaded_manifests = Dict(
+ title=_("A dict of manifest information per build."),
+ key_type=Int(), value_type=Dict(),
+ required=False, readonly=True)
+
+ def addUploadedManifest(build_id, manifest_info):
+ """Add the manifest information for one of the builds in this
+ BuildRequest.
+ """
+
class IOCIRecipeView(Interface):
"""`IOCIRecipe` attributes that require launchpad.View permission."""
diff --git a/lib/lp/oci/interfaces/ocirecipejob.py b/lib/lp/oci/interfaces/ocirecipejob.py
index 104a2ff..2b59faf 100644
--- a/lib/lp/oci/interfaces/ocirecipejob.py
+++ b/lib/lp/oci/interfaces/ocirecipejob.py
@@ -19,6 +19,8 @@ from zope.interface import (
)
from zope.schema import (
Datetime,
+ Dict,
+ Int,
List,
TextLine,
)
@@ -78,6 +80,16 @@ class IOCIRecipeRequestBuildsJob(IRunnableJob):
error_message = TextLine(
title=_("Error message"), required=False, readonly=True)
+ uploaded_manifests = Dict(
+ title=_("A dict of manifest information per build."),
+ key_type=Int(), value_type=Dict(),
+ required=False, readonly=True)
+
+ def addUploadedManifest(build_id, manifest_info):
+ """Add the manifest information for one of the builds in this
+ BuildRequest.
+ """
+
class IOCIRecipeRequestBuildsJobSource(IJobSource):
diff --git a/lib/lp/oci/interfaces/ociregistryclient.py b/lib/lp/oci/interfaces/ociregistryclient.py
index 184a93f..71af23a 100644
--- a/lib/lp/oci/interfaces/ociregistryclient.py
+++ b/lib/lp/oci/interfaces/ociregistryclient.py
@@ -54,3 +54,6 @@ class IOCIRegistryClient(Interface):
:param build: The `IOCIRecipeBuild` to upload.
"""
+
+ def uploadManifestList(build_request):
+ """Upload the "fat manifest" which aggregates all platforms built."""
diff --git a/lib/lp/oci/model/ocirecipe.py b/lib/lp/oci/model/ocirecipe.py
index 15fa5be..e4b46e3 100644
--- a/lib/lp/oci/model/ocirecipe.py
+++ b/lib/lp/oci/model/ocirecipe.py
@@ -38,7 +38,10 @@ from zope.component import (
from zope.event import notify
from zope.interface import implementer
from zope.security.interfaces import Unauthorized
-from zope.security.proxy import removeSecurityProxy
+from zope.security.proxy import (
+ isinstance as zope_isinstance,
+ removeSecurityProxy,
+ )
from lp.app.interfaces.launchpad import ILaunchpadCelebrities
from lp.app.interfaces.security import IAuthorization
@@ -695,6 +698,13 @@ class OCIRecipeBuildRequest:
return self.job.date_finished
@property
+ def uploaded_manifests(self):
+ return self.job.uploaded_manifests
+
+ def addUploadedManifest(self, build_id, manifest_info):
+ self.job.addUploadedManifest(build_id, manifest_info)
+
+ @property
def status(self):
status_map = {
JobStatus.WAITING: OCIRecipeBuildRequestStatus.PENDING,
@@ -712,3 +722,8 @@ class OCIRecipeBuildRequest:
@property
def builds(self):
return self.job.builds
+
+ def __eq__(self, other):
+ if not zope_isinstance(other, self.__class__):
+ return False
+ return self.id == other.id
diff --git a/lib/lp/oci/model/ocirecipebuildjob.py b/lib/lp/oci/model/ocirecipebuildjob.py
index 73fa4d6..0118d48 100644
--- a/lib/lp/oci/model/ocirecipebuildjob.py
+++ b/lib/lp/oci/model/ocirecipebuildjob.py
@@ -41,8 +41,12 @@ from lp.oci.interfaces.ociregistryclient import (
OCIRegistryError,
)
from lp.services.database.enumcol import DBEnum
-from lp.services.database.interfaces import IStore
+from lp.services.database.interfaces import (
+ IMasterStore,
+ IStore,
+ )
from lp.services.database.stormbase import StormBase
+from lp.services.job.interfaces.job import JobStatus
from lp.services.job.model.job import (
EnumeratedSubclass,
Job,
@@ -227,6 +231,37 @@ class OCIRegistryUploadJob(OCIRecipeBuildJobDerived):
"""See `IOCIRegistryUploadJob`."""
self.json_data["errors"] = errors
+ def allBuildsUploaded(self, build_request):
+ """Returns True if all builds of the given build_request already
+ finished uploading. False otherwise.
+
+ Note that this method locks all upload jobs at database level,
+ preventing them from updating their status until the end of the
+ current transaction. Use it with caution.
+ """
+ builds = list(build_request.builds)
+ uploads_per_build = {i: list(i.registry_upload_jobs) for i in builds}
+ upload_jobs = sum(uploads_per_build.values(), [])
+
+ # Lock the Job rows, so no other job updates it's status until the
+ # end of this job's transaction. This is done to avoid race conditions,
+ # where 2 upload jobs could be running simultaneously and none of them
+ # realises that is the last upload.
+ store = IMasterStore(builds[0])
+ placeholders = ', '.join('?' for _ in upload_jobs)
+ sql = (
+ "SELECT id, status FROM job WHERE id IN (%s) FOR UPDATE"
+ % placeholders)
+ store.execute(sql, [i.job_id for i in upload_jobs])
+
+ for build, upload_jobs in uploads_per_build.items():
+ has_finished_upload = any(
+ i.status == JobStatus.COMPLETED or i.job_id == self.job_id
+ for i in upload_jobs)
+ if not has_finished_upload:
+ return False
+ return True
+
def run(self):
"""See `IRunnableJob`."""
client = getUtility(IOCIRegistryClient)
@@ -235,6 +270,12 @@ class OCIRegistryUploadJob(OCIRecipeBuildJobDerived):
try:
try:
client.upload(self.build)
+ # The "allBuildsUploaded" call will lock, on the database,
+ # all upload jobs for update until this transaction finishes.
+ # So, make sure this is the last thing being done by this job.
+ build_request = self.build.build_request
+ if build_request and self.allBuildsUploaded(build_request):
+ client.uploadManifestList(self.build.build_request)
except OCIRegistryError as e:
self.error_summary = str(e)
self.errors = e.errors
diff --git a/lib/lp/oci/model/ocirecipejob.py b/lib/lp/oci/model/ocirecipejob.py
index 15eac4f..3320cff 100644
--- a/lib/lp/oci/model/ocirecipejob.py
+++ b/lib/lp/oci/model/ocirecipejob.py
@@ -169,6 +169,8 @@ class OCIRecipeRequestBuildsJob(OCIRecipeJobDerived):
"requester": requester.id,
"architectures": (
list(architectures) if architectures is not None else None),
+ # A dict of build_id: manifest location
+ "uploaded_manifests": {}
}
recipe_job = OCIRecipeJob(recipe, cls.class_job_type, metadata)
job = cls(recipe_job)
@@ -258,6 +260,16 @@ class OCIRecipeRequestBuildsJob(OCIRecipeJobDerived):
architectures = self.metadata["architectures"]
return set(architectures) if architectures is not None else None
+ @property
+ def uploaded_manifests(self):
+ return {
+ # Converts keys to integer since saving json to database
+ # converts them to strings.
+ int(k): v for k, v in self.metadata["uploaded_manifests"].items()}
+
+ def addUploadedManifest(self, build_id, manifest_info):
+ self.metadata["uploaded_manifests"][int(build_id)] = manifest_info
+
def run(self):
"""See `IRunnableJob`."""
requester = self.requester
diff --git a/lib/lp/oci/model/ociregistryclient.py b/lib/lp/oci/model/ociregistryclient.py
index 7561267..55f61ce 100644
--- a/lib/lp/oci/model/ociregistryclient.py
+++ b/lib/lp/oci/model/ociregistryclient.py
@@ -15,6 +15,8 @@ from functools import partial
import hashlib
from io import BytesIO
import json
+
+
try:
from json.decoder import JSONDecodeError
except ImportError:
@@ -229,17 +231,59 @@ class OCIRegistryClient:
"""
# XXX twom 2020-04-17 This needs to include OCIProjectSeries and
# base image name
-
return "{}".format("edge")
@classmethod
+ def _uploadRegistryManifest(cls, http_client, registry_manifest,
+ push_rule, build=None):
+ """Uploads the build manifest, returning its content information.
+
+ The returned information can be used to create a Manifest list
+ including the uploaded manifest, for example, in order to create
+ multi-architecture images.
+
+ :return: A dict with {"digest": "sha256:xxx", "size": total_bytes}
+ """
+ digest = None
+ data = json.dumps(registry_manifest)
+ size = len(data)
+ content_type = registry_manifest.get(
+ "mediaType",
+ "application/vnd.docker.distribution.manifest.v2+json")
+ if build is None:
+ # When uploading a manifest list, use the tag.
+ tag = cls._calculateTag(build, push_rule)
+ else:
+ # When uploading individual build manifests, use their digest.
+ tag = "sha256:%s" % hashlib.sha256(data).hexdigest()
+ try:
+ manifest_response = http_client.requestPath(
+ "/manifests/{}".format(tag),
+ data=data,
+ headers={"Content-Type": content_type},
+ method="PUT")
+ digest = manifest_response.headers.get("Docker-Content-Digest")
+ except HTTPError as http_error:
+ manifest_response = http_error.response
+ if manifest_response.status_code != 201:
+ if build:
+ msg = "Failed to upload manifest for {} ({}) in {}".format(
+ build.recipe.name, push_rule.registry_url, build.id)
+ else:
+ msg = ("Failed to upload manifest of manifests for"
+ " {} ({})").format(
+ push_rule.recipe.name, push_rule.registry_url)
+ raise cls._makeRegistryError(
+ ManifestUploadFailed, msg, manifest_response)
+ return {"digest": digest, "size": size}
+
+ @classmethod
def _upload_to_push_rule(
cls, push_rule, build, manifest, digests, preloaded_data):
http_client = RegistryHTTPClient.getInstance(push_rule)
for section in manifest:
- # Work out names and tags
- tag = cls._calculateTag(build, push_rule)
+ # Work out names
file_data = preloaded_data[section["Config"]]
config = file_data["config_file"]
# Upload the layers involved
@@ -269,24 +313,13 @@ class OCIRegistryClient:
layer_sizes)
# Upload the registry manifest
- try:
- manifest_response = http_client.requestPath(
- "/manifests/{}".format(tag),
- json=registry_manifest,
- headers={
- "Content-Type":
- "application/"
- "vnd.docker.distribution.manifest.v2+json"
- },
- method="PUT")
- except HTTPError as http_error:
- manifest_response = http_error.response
- if manifest_response.status_code != 201:
- raise cls._makeRegistryError(
- ManifestUploadFailed,
- "Failed to upload manifest for {} ({}) in {}".format(
- build.recipe.name, push_rule.registry_url, build.id),
- manifest_response)
+ maanifest = cls._uploadRegistryManifest(
+ http_client, registry_manifest, push_rule, build)
+
+ # Save the uploaded manifest location, so we can use it in case
+ # this is a multi-arch image upload.
+ if build.build_request:
+ build.build_request.addUploadedManifest(build.id, maanifest)
@classmethod
def upload(cls, build):
@@ -318,6 +351,43 @@ class OCIRegistryClient:
elif len(exceptions) > 1:
raise MultipleOCIRegistryError(exceptions)
+ @classmethod
+ def makeMultiArchManifest(cls, build_request):
+ """Returns the multi-arch manifest content including all builds of
+ the given build_request.
+ """
+ manifests = []
+ for build in build_request.builds:
+ build_manifest = build_request.uploaded_manifests.get(build.id)
+ if not build_manifest:
+ continue
+ digest = build_manifest["digest"]
+ size = build_manifest["size"]
+ arch = build.processor.name
+ manifests.append({
+ "mediaType": ("application/"
+ "vnd.docker.distribution.manifest.v2+json"),
+ "size": size,
+ "digest": digest,
+ "platform": {"architecture": arch, "os": "linux"}
+ })
+ return {
+ "schemaVersion": 2,
+ "mediaType": ("application/"
+ "vnd.docker.distribution.manifest.list.v2+json"),
+ "manifests": manifests}
+
+ @classmethod
+ def uploadManifestList(cls, build_request):
+ """Uploads to all build_request.recipe.push_rules the manifest list
+ for the builds in the given build_request.
+ """
+ multi_manifest_content = cls.makeMultiArchManifest(build_request)
+ for push_rule in build_request.recipe.push_rules:
+ http_client = RegistryHTTPClient.getInstance(push_rule)
+ cls._uploadRegistryManifest(
+ http_client, multi_manifest_content, push_rule, build=None)
+
class OCIRegistryAuthenticationError(Exception):
def __init__(self, msg, http_error=None):
@@ -436,8 +506,8 @@ class BearerTokenRegistryClient(RegistryHTTPClient):
:param auth_retry: Should we authenticate and retry the request if
it fails with HTTP 401 code?"""
+ headers = request_kwargs.pop("headers", {})
try:
- headers = request_kwargs.pop("headers", {})
if self.auth_token is not None:
headers["Authorization"] = "Bearer %s" % self.auth_token
return proxy_urlfetch(url, headers=headers, **request_kwargs)
@@ -445,5 +515,6 @@ class BearerTokenRegistryClient(RegistryHTTPClient):
if auth_retry and e.response.status_code == 401:
self.authenticate(e.response)
return self.request(
- url, auth_retry=False, *args, **request_kwargs)
+ url, auth_retry=False, headers=headers,
+ *args, **request_kwargs)
raise
diff --git a/lib/lp/oci/tests/test_ocirecipebuildjob.py b/lib/lp/oci/tests/test_ocirecipebuildjob.py
index fd3e0ff..f497c03 100644
--- a/lib/lp/oci/tests/test_ocirecipebuildjob.py
+++ b/lib/lp/oci/tests/test_ocirecipebuildjob.py
@@ -14,11 +14,14 @@ from testtools.matchers import (
MatchesDict,
MatchesListwise,
MatchesStructure,
+ Not,
)
import transaction
+from zope.component import getUtility
from zope.interface import implementer
from lp.buildmaster.enums import BuildStatus
+from lp.buildmaster.interfaces.processor import IProcessorSet
from lp.oci.interfaces.ocirecipe import (
OCI_RECIPE_ALLOW_CREATE,
OCI_RECIPE_WEBHOOKS_FEATURE_FLAG,
@@ -27,6 +30,7 @@ from lp.oci.interfaces.ocirecipebuildjob import (
IOCIRecipeBuildJob,
IOCIRegistryUploadJob,
)
+from lp.oci.interfaces.ocirecipejob import IOCIRecipeRequestBuildsJobSource
from lp.oci.interfaces.ociregistryclient import (
BlobUploadFailed,
IOCIRegistryClient,
@@ -42,7 +46,10 @@ from lp.services.features.testing import FeatureFixture
from lp.services.job.runner import JobRunner
from lp.services.webapp import canonical_url
from lp.services.webhooks.testing import LogsScheduledWebhooks
-from lp.testing import TestCaseWithFactory
+from lp.testing import (
+ admin_logged_in,
+ TestCaseWithFactory,
+ )
from lp.testing.dbuser import dbuser
from lp.testing.fakemethod import FakeMethod
from lp.testing.fixture import ZopeUtilityFixture
@@ -69,6 +76,7 @@ class FakeRegistryClient:
def __init__(self):
self.upload = FakeMethod()
+ self.uploadManifestList = FakeMethod()
class FakeOCIBuildJob(OCIRecipeBuildJobDerived):
@@ -122,10 +130,13 @@ class TestOCIRegistryUploadJob(TestCaseWithFactory):
ocibuild = self.factory.makeOCIRecipeBuild(
builder=self.factory.makeBuilder(), **kwargs)
ocibuild.updateStatus(BuildStatus.FULLYBUILT)
- self.factory.makeWebhook(
- target=ocibuild.recipe, event_types=["oci-recipe:build:0.1"])
+ self.makeWebhook(ocibuild.recipe)
return ocibuild
+ def makeWebhook(self, recipe):
+ self.factory.makeWebhook(
+ target=recipe, event_types=["oci-recipe:build:0.1"])
+
def assertWebhookDeliveries(self, ocibuild,
expected_registry_upload_statuses, logger):
hook = ocibuild.recipe.webhooks.one()
@@ -137,7 +148,8 @@ class TestOCIRegistryUploadJob(TestCaseWithFactory):
"action": Equals("status-changed"),
"recipe": Equals(
canonical_url(ocibuild.recipe, force_local_path=True)),
- "build_request": Is(None),
+ "build_request": (Is(None) if ocibuild.build_request is None
+ else Not(Is(None))),
"status": Equals("Successfully built"),
"registry_upload_status": Equals(expected),
} for expected in expected_registry_upload_statuses]
@@ -165,22 +177,94 @@ class TestOCIRegistryUploadJob(TestCaseWithFactory):
job = OCIRegistryUploadJob.create(ocibuild)
self.assertProvides(job, IOCIRegistryUploadJob)
+ def makeRecipe(self, include_i386=True, include_amd64=True):
+ i386 = getUtility(IProcessorSet).getByName("386")
+ amd64 = getUtility(IProcessorSet).getByName("amd64")
+ recipe = self.factory.makeOCIRecipe()
+ distroseries = self.factory.makeDistroSeries(
+ distribution=recipe.oci_project.distribution)
+ distro_i386 = self.factory.makeDistroArchSeries(
+ distroseries=distroseries, architecturetag="i386",
+ processor=i386)
+ distro_i386.addOrUpdateChroot(self.factory.makeLibraryFileAlias())
+ distro_amd64 = self.factory.makeDistroArchSeries(
+ distroseries=distroseries, architecturetag="amd64",
+ processor=amd64)
+ distro_amd64.addOrUpdateChroot(self.factory.makeLibraryFileAlias())
+
+ archs = []
+ if include_i386:
+ archs.append(i386)
+ if include_amd64:
+ archs.append(amd64)
+ recipe.setProcessors(archs)
+ return recipe
+
def test_run(self):
logger = self.useFixture(FakeLogger())
- ocibuild = self.makeOCIRecipeBuild()
+ recipe = self.makeRecipe(include_i386=False, include_amd64=True)
+ # Creates a build request with a build in it.
+ build_request = recipe.requestBuilds(recipe.owner)
+ with admin_logged_in():
+ jobs = getUtility(IOCIRecipeRequestBuildsJobSource).iterReady()
+ with dbuser(config.IOCIRecipeRequestBuildsJobSource.dbuser):
+ JobRunner(jobs).runAll()
+
+ self.assertEqual(1, build_request.builds.count())
+ ocibuild = build_request.builds[0]
+ ocibuild.updateStatus(BuildStatus.FULLYBUILT)
+ self.makeWebhook(recipe)
+
self.assertContentEqual([], ocibuild.registry_upload_jobs)
job = OCIRegistryUploadJob.create(ocibuild)
client = FakeRegistryClient()
self.useFixture(ZopeUtilityFixture(client, IOCIRegistryClient))
with dbuser(config.IOCIRegistryUploadJobSource.dbuser):
run_isolated_jobs([job])
+
self.assertEqual([((ocibuild,), {})], client.upload.calls)
+ self.assertEqual([((build_request,), {})],
+ client.uploadManifestList.calls)
self.assertContentEqual([job], ocibuild.registry_upload_jobs)
self.assertIsNone(job.error_summary)
self.assertIsNone(job.errors)
self.assertEqual([], pop_notifications())
self.assertWebhookDeliveries(ocibuild, ["Pending", "Uploaded"], logger)
+ def test_run_multiple_architectures(self):
+ recipe = self.makeRecipe()
+
+ # Creates a build request with builds.
+ build_request = recipe.requestBuilds(recipe.owner)
+ with admin_logged_in():
+ jobs = getUtility(IOCIRecipeRequestBuildsJobSource).iterReady()
+ with dbuser(config.IOCIRecipeRequestBuildsJobSource.dbuser):
+ JobRunner(jobs).runAll()
+
+ builds = build_request.builds
+ self.assertEqual(2, builds.count())
+ self.assertEqual(builds[0].build_request, builds[1].build_request)
+
+ upload_jobs = []
+ for build in builds:
+ self.assertContentEqual([], build.registry_upload_jobs)
+ upload_jobs.append(OCIRegistryUploadJob.create(build))
+
+ client = FakeRegistryClient()
+ self.useFixture(ZopeUtilityFixture(client, IOCIRegistryClient))
+
+ with dbuser(config.IOCIRegistryUploadJobSource.dbuser):
+ JobRunner([upload_jobs[0]]).runAll()
+ self.assertEqual([((builds[0],), {})], client.upload.calls)
+ self.assertEqual([], client.uploadManifestList.calls)
+
+ with dbuser(config.IOCIRegistryUploadJobSource.dbuser):
+ JobRunner([upload_jobs[1]]).runAll()
+ self.assertEqual(
+ [((builds[0],), {}), ((builds[1],), {})], client.upload.calls)
+ self.assertEqual([((builds[1].build_request, ), {})],
+ client.uploadManifestList.calls)
+
def test_run_failed_registry_error(self):
# A run that fails with a registry error sets the registry upload
# status to FAILED, and stores the detailed errors.
diff --git a/lib/lp/oci/tests/test_ociregistryclient.py b/lib/lp/oci/tests/test_ociregistryclient.py
index 3c5aa8c..c73cb0c 100644
--- a/lib/lp/oci/tests/test_ociregistryclient.py
+++ b/lib/lp/oci/tests/test_ociregistryclient.py
@@ -11,6 +11,7 @@ from functools import partial
import io
import json
import os
+import re
import tarfile
import uuid
@@ -33,13 +34,17 @@ from testtools.matchers import (
Raises,
)
import transaction
+from zope.component._api import getUtility
from zope.security.proxy import removeSecurityProxy
+from lp.buildmaster.interfaces.processor import IProcessorSet
+from lp.oci.interfaces.ocirecipejob import IOCIRecipeRequestBuildsJobSource
from lp.oci.interfaces.ociregistryclient import (
BlobUploadFailed,
ManifestUploadFailed,
MultipleOCIRegistryError,
)
+from lp.oci.model.ocirecipe import OCIRecipeBuildRequest
from lp.oci.model.ociregistryclient import (
BearerTokenRegistryClient,
OCIRegistryAuthenticationError,
@@ -50,6 +55,7 @@ from lp.oci.model.ociregistryclient import (
from lp.oci.tests.helpers import OCIConfigHelperMixin
from lp.services.compat import mock
from lp.testing import TestCaseWithFactory
+from lp.testing.fixture import ZopeUtilityFixture
from lp.testing.layers import (
DatabaseFunctionalLayer,
LaunchpadZopelessLayer,
@@ -142,6 +148,23 @@ class TestOCIRegistryClient(OCIConfigHelperMixin, SpyProxyCallsMixin,
transaction.commit()
+ def addManifestResponses(self, push_rule, status_code=201, json=None):
+ """Add responses for manifest upload URLs."""
+ # PUT to "anonymous" architecture-specific manifest.
+ manifests_url = "{}/v2/{}/manifests/sha256:.*".format(
+ push_rule.registry_credentials.url,
+ push_rule.image_name
+ )
+ responses.add(
+ "PUT", re.compile(manifests_url), status=status_code, json=json)
+
+ # PUT to tagged multi-arch manifest.
+ manifests_url = "{}/v2/{}/manifests/edge".format(
+ push_rule.registry_credentials.url,
+ push_rule.image_name
+ )
+ responses.add("PUT", manifests_url, status=status_code, json=json)
+
@responses.activate
def test_upload(self):
self._makeFiles()
@@ -154,11 +177,7 @@ class TestOCIRegistryClient(OCIConfigHelperMixin, SpyProxyCallsMixin,
push_rule = self.build.recipe.push_rules[0]
responses.add("GET", "%s/v2/" % push_rule.registry_url, status=200)
- manifests_url = "{}/v2/{}/manifests/edge".format(
- push_rule.registry_credentials.url,
- push_rule.image_name
- )
- responses.add("PUT", manifests_url, status=201)
+ self.addManifestResponses(push_rule)
self.client.upload(self.build)
@@ -206,9 +225,7 @@ class TestOCIRegistryClient(OCIConfigHelperMixin, SpyProxyCallsMixin,
push_rule = self.build.recipe.push_rules[0]
responses.add("GET", "%s/v2/" % push_rule.registry_url, status=200)
- manifests_url = "{}/v2/{}/manifests/edge".format(
- push_rule.registry_credentials.url, push_rule.image_name)
- responses.add("PUT", manifests_url, status=201)
+ self.addManifestResponses(push_rule)
self.client.upload(self.build)
@@ -237,10 +254,8 @@ class TestOCIRegistryClient(OCIConfigHelperMixin, SpyProxyCallsMixin,
responses.add(
"GET", "%s/v2/" % push_rule.registry_url, status=200)
- manifests_url = "{}/v2/{}/manifests/edge".format(
- push_rule.registry_credentials.url, push_rule.image_name)
status = 400 if i < 2 else 201
- responses.add("PUT", manifests_url, status=status)
+ self.addManifestResponses(push_rule, status_code=status)
error = self.assertRaises(
MultipleOCIRegistryError, self.client.upload, self.build)
@@ -472,9 +487,6 @@ class TestOCIRegistryClient(OCIConfigHelperMixin, SpyProxyCallsMixin,
responses.add(
"GET", "{}/v2/".format(push_rule.registry_url), status=200)
- manifests_url = "{}/v2/{}/manifests/edge".format(
- push_rule.registry_credentials.url,
- push_rule.image_name)
put_errors = [
{
"code": "MANIFEST_INVALID",
@@ -482,8 +494,8 @@ class TestOCIRegistryClient(OCIConfigHelperMixin, SpyProxyCallsMixin,
"detail": [],
},
]
- responses.add(
- "PUT", manifests_url, status=400, json={"errors": put_errors})
+ self.addManifestResponses(
+ push_rule, status_code=400, json={"errors": put_errors})
expected_msg = "Failed to upload manifest for {} ({}) in {}".format(
self.build.recipe.name, self.push_rule.registry_url, self.build.id)
@@ -509,10 +521,7 @@ class TestOCIRegistryClient(OCIConfigHelperMixin, SpyProxyCallsMixin,
responses.add(
"GET", "{}/v2/".format(push_rule.registry_url), status=200)
- manifests_url = "{}/v2/{}/manifests/edge".format(
- push_rule.registry_credentials.url,
- push_rule.image_name)
- responses.add("PUT", manifests_url, status=200)
+ self.addManifestResponses(push_rule, status_code=200)
expected_msg = "Failed to upload manifest for {} ({}) in {}".format(
self.build.recipe.name, self.push_rule.registry_url, self.build.id)
@@ -526,6 +535,64 @@ class TestOCIRegistryClient(OCIConfigHelperMixin, SpyProxyCallsMixin,
Equals(expected_msg)),
MatchesStructure(errors=Is(None))))))
+ @responses.activate
+ def test_multi_arch_manifest_upload(self):
+ """Ensure that multi-arch manifest upload works and tags correctly
+ the uploaded image."""
+ # Creates a build request with 2 builds.
+ recipe = self.build.recipe
+ build1 = self.build
+ build2 = self.factory.makeOCIRecipeBuild(
+ recipe=recipe)
+ naked_build1 = removeSecurityProxy(build1)
+ naked_build2 = removeSecurityProxy(build1)
+ naked_build1.processor = getUtility(IProcessorSet).getByName('386')
+ naked_build2.processor = getUtility(IProcessorSet).getByName('amd64')
+
+ # Creates a mock IOCIRecipeRequestBuildsJobSource, as it was created
+ # by the celery job and triggered the 2 registry uploads already.
+ job = mock.Mock()
+ job.builds = [build1, build2]
+ job.uploaded_manifests = {
+ build1.id: {"digest": "build1digest", "size": 123},
+ build2.id: {"digest": "build2digest", "size": 321},
+ }
+ job_source = mock.Mock()
+ job_source.getByOCIRecipeAndID.return_value = job
+ self.useFixture(
+ ZopeUtilityFixture(job_source, IOCIRecipeRequestBuildsJobSource))
+ build_request = OCIRecipeBuildRequest(recipe, -1)
+
+ push_rule = self.build.recipe.push_rules[0]
+ responses.add(
+ "GET", "{}/v2/".format(push_rule.registry_url), status=200)
+ self.addManifestResponses(push_rule, status_code=201)
+
+ self.client.uploadManifestList(build_request)
+ self.assertEqual(2, len(responses.calls))
+ auth_call, manifest_call = responses.calls
+ self.assertEndsWith(
+ manifest_call.request.url,
+ "/v2/%s/manifests/edge" % push_rule.image_name)
+ self.assertEqual({
+ "schemaVersion": 2,
+ "mediaType": "application/"
+ "vnd.docker.distribution.manifest.list.v2+json",
+ "manifests": [{
+ "platform": {"os": "linux", "architecture": "amd64"},
+ "mediaType": "application/"
+ "vnd.docker.distribution.manifest.v2+json",
+ "digest": "build1digest",
+ "size": 123
+ }, {
+ "platform": {"os": "linux", "architecture": "386"},
+ "mediaType": "application/"
+ "vnd.docker.distribution.manifest.v2+json",
+ "digest": "build2digest",
+ "size": 321
+ }]
+ }, json.loads(manifest_call.request.body))
+
class TestRegistryHTTPClient(OCIConfigHelperMixin, SpyProxyCallsMixin,
TestCaseWithFactory):
