launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #24588
[Merge] ~twom/launchpad:oci-registry-upload into launchpad:master
Tom Wardill has proposed merging ~twom/launchpad:oci-registry-upload into launchpad:master.
Commit message:
Upload built images to an OCI registry
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~twom/launchpad/+git/launchpad/+merge/381981
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~twom/launchpad:oci-registry-upload into launchpad:master.
diff --git a/database/schema/security.cfg b/database/schema/security.cfg
index 942c5b4..55226e1 100644
--- a/database/schema/security.cfg
+++ b/database/schema/security.cfg
@@ -999,6 +999,7 @@ public.livefsfile = SELECT
public.ocifile = SELECT
public.ociproject = SELECT
public.ociprojectname = SELECT
+public.ocipushrule = SELECT
public.ocirecipe = SELECT
public.ocirecipebuild = SELECT, UPDATE
public.ocirecipebuildjob = SELECT, INSERT
@@ -1439,6 +1440,7 @@ public.ocifile = SELECT, INSERT
public.ociproject = SELECT
public.ociprojectname = SELECT
public.ociprojectseries = SELECT
+public.ocipushrule = SELECT
public.ocirecipe = SELECT, UPDATE
public.ocirecipebuild = SELECT, UPDATE
public.ocirecipebuildjob = SELECT, INSERT, UPDATE
@@ -2678,3 +2680,42 @@ public.teammembership = SELECT
public.teamparticipation = SELECT
public.webhook = SELECT
public.webhookjob = SELECT, INSERT
+
+[oci-build-job]
+type=user
+groups=script
+public.account = SELECT
+public.archive = SELECT
+public.branch = SELECT
+public.builder = SELECT
+public.buildfarmjob = SELECT, INSERT
+public.buildqueue = SELECT, INSERT, UPDATE
+public.distribution = SELECT
+public.distroarchseries = SELECT
+public.distroseries = SELECT
+public.emailaddress = SELECT
+public.gitref = SELECT
+public.gitrepository = SELECT
+public.job = SELECT, INSERT, UPDATE
+public.libraryfilealias = SELECT
+public.libraryfilecontent = SELECT
+public.person = SELECT
+public.personsettings = SELECT
+public.pocketchroot = SELECT
+public.processor = SELECT
+public.product = SELECT
+public.ocirecipe = SELECT, UPDATE
+public.ocirecipearch = SELECT
+public.ocirecipebuild = SELECT, INSERT, UPDATE
+public.ocirecipebuildjob = SELECT, UPDATE
+public.ocifile = SELECT
+public.ociproject = SELECT
+public.ociprojectname = SELECT
+public.ociprojectseries = SELECT
+public.ocipushrule = SELECT
+public.ociregistrycredentials = SELECT
+public.sourcepackagename = SELECT
+public.teammembership = SELECT
+public.teamparticipation = SELECT
+public.webhook = SELECT
+public.webhookjob = SELECT, INSERT
diff --git a/lib/lp/oci/configure.zcml b/lib/lp/oci/configure.zcml
index 565412d..7c2ead7 100644
--- a/lib/lp/oci/configure.zcml
+++ b/lib/lp/oci/configure.zcml
@@ -127,4 +127,22 @@
interface="lp.oci.interfaces.ocipushrule.IOCIPushRuleSet"/>
</securedutility>
+ <!-- OCI related jobs -->
+ <securedutility
+ component="lp.oci.model.ocirecipebuildjob.OCIRegistryUploadJob"
+ provides="lp.oci.interfaces.ocirecipebuildjob.IOCIRegistryUploadJobSource">
+ <allow interface="lp.oci.interfaces.ocirecipebuildjob.IOCIRegistryUploadJobSource" />
+ </securedutility>
+ <class class="lp.oci.model.ocirecipebuildjob.OCIRegistryUploadJob">
+ <allow interface="lp.oci.interfaces.ocirecipebuildjob.IOCIRecipeBuildJob" />
+ <allow interface="lp.oci.interfaces.ocirecipebuildjob.IOCIRegistryUploadJob" />
+ </class>
+
+ <!-- Registry interaction -->
+ <securedutility
+ class="lp.oci.model.ociregistryclient.OCIRegistryClient"
+ provides="lp.oci.interfaces.ociregistryclient.IOCIRegistryClient">
+ <allow interface="lp.oci.interfaces.ociregistryclient.IOCIRegistryClient" />
+ </securedutility>
+
</configure>
diff --git a/lib/lp/oci/interfaces/ocirecipe.py b/lib/lp/oci/interfaces/ocirecipe.py
index 25a8967..eda2782 100644
--- a/lib/lp/oci/interfaces/ocirecipe.py
+++ b/lib/lp/oci/interfaces/ocirecipe.py
@@ -214,6 +214,12 @@ class IOCIRecipeView(Interface):
# Really IOCIPushRule, patched in _schema_cirular_imports.
value_type=Reference(schema=Interface), readonly=True)
+ can_upload_to_registry = Bool(
+ title=_("Can upload to registry"), required=True, readonly=True,
+ description=_(
+ "Whether everything is set up to allow uploading builds of "
+ "this OCI recipe to a registry."))
+
class IOCIRecipeEdit(IWebhookTarget):
"""`IOCIRecipe` methods that require launchpad.Edit permission."""
diff --git a/lib/lp/oci/interfaces/ocirecipebuild.py b/lib/lp/oci/interfaces/ocirecipebuild.py
index 752e61d..58968a5 100644
--- a/lib/lp/oci/interfaces/ocirecipebuild.py
+++ b/lib/lp/oci/interfaces/ocirecipebuild.py
@@ -12,8 +12,14 @@ __all__ = [
'IOCIRecipeBuildSet',
]
-from lazr.restful.fields import Reference
-from zope.interface import Interface
+from lazr.restful.fields import (
+ CollectionField,
+ Reference,
+ )
+from zope.interface import (
+ Attribute,
+ Interface,
+ )
from zope.schema import (
Bool,
Datetime,
@@ -77,6 +83,7 @@ class IOCIRecipeBuildView(IPackageBuild):
title=_("The series and architecture for which to build."),
required=True, readonly=True)
+<<<<<<< lib/lp/oci/interfaces/ocirecipebuild.py
score = Int(
title=_("Score of the related build farm job (if any)."),
required=False, readonly=True)
@@ -116,6 +123,17 @@ class IOCIRecipeBuildEdit(Interface):
If the build is not in a cancellable state, this method is a no-op.
"""
+=======
+ manifest = Attribute(_("The manifest of the image."))
+
+ digests = Attribute(_("File containing the image digests."))
+
+ registry_upload_jobs = CollectionField(
+ title=_("Registry upload jobs for this build."),
+ # Really IOCIRegistryUploadJob.
+ value_type=Reference(schema=Interface),
+ readonly=True)
+>>>>>>> lib/lp/oci/interfaces/ocirecipebuild.py
class IOCIRecipeBuildAdmin(Interface):
diff --git a/lib/lp/oci/interfaces/ocirecipebuildjob.py b/lib/lp/oci/interfaces/ocirecipebuildjob.py
index 6c25b73..1cde82e 100644
--- a/lib/lp/oci/interfaces/ocirecipebuildjob.py
+++ b/lib/lp/oci/interfaces/ocirecipebuildjob.py
@@ -8,17 +8,30 @@ from __future__ import absolute_import, print_function, unicode_literals
__metaclass__ = type
__all__ = [
'IOCIRecipeBuildJob',
+ 'IOCIRecipeBuildRegistryUploadStatusChangedEvent',
+ 'IOCIRegistryUploadJob',
+ 'IOCIRegistryUploadJobSource',
]
from lazr.restful.fields import Reference
+from zope.component.interfaces import IObjectEvent
from zope.interface import (
Attribute,
Interface,
)
+from zope.schema import TextLine
from lp import _
from lp.oci.interfaces.ocirecipebuild import IOCIRecipeBuild
-from lp.services.job.interfaces.job import IJob
+from lp.services.job.interfaces.job import (
+ IJob,
+ IJobSource,
+ IRunnableJob,
+ )
+
+
+class IOCIRecipeBuildRegistryUploadStatusChangedEvent(IObjectEvent):
+ """The store upload status of an OCI recipe build changed."""
class IOCIRecipeBuildJob(Interface):
@@ -32,3 +45,19 @@ class IOCIRecipeBuildJob(Interface):
schema=IOCIRecipeBuild, required=True, readonly=True)
json_data = Attribute(_("A dict of data about the job."))
+
+
+class IOCIRegistryUploadJob(IRunnableJob):
+ """A Job that uploads an OCI image to a registry."""
+
+ error_message = TextLine(
+ title=_("Error message"), required=False, readonly=True)
+
+
+class IOCIRegistryUploadJobSource(IJobSource):
+
+ def create(build):
+ """Upload an OCI image to a registry.
+
+ :param build: The OCI recipe build to upload.
+ """
diff --git a/lib/lp/oci/interfaces/ociregistryclient.py b/lib/lp/oci/interfaces/ociregistryclient.py
new file mode 100644
index 0000000..efcd50f
--- /dev/null
+++ b/lib/lp/oci/interfaces/ociregistryclient.py
@@ -0,0 +1,33 @@
+# Copyright 2020 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Interface for communication with an OCI registry."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+ 'BlobUploadFailed',
+ 'IOCIRegistryClient',
+ 'ManifestUploadFailed',
+]
+
+from zope.interface import Interface
+
+
+class BlobUploadFailed(Exception):
+ pass
+
+
+class ManifestUploadFailed(Exception):
+ pass
+
+
+class IOCIRegistryClient(Interface):
+ """Interface for the API provided by an OCI registry."""
+
+ def upload(ocibuild):
+ """Upload an OCI image to a registry.
+
+ :param ocibuild: The `IOCIRecipeBuild` to upload.
+ """
diff --git a/lib/lp/oci/model/ocirecipe.py b/lib/lp/oci/model/ocirecipe.py
index 4e3273a..e69c290 100644
--- a/lib/lp/oci/model/ocirecipe.py
+++ b/lib/lp/oci/model/ocirecipe.py
@@ -377,6 +377,10 @@ class OCIRecipe(Storm, WebhookTargetMixin):
order_by = Desc(OCIRecipeBuild.id)
return self._getBuilds(filter_term, order_by)
+ @property
+ def can_upload_to_registry(self):
+ return bool(self.push_rules.count())
+
class OCIRecipeArch(Storm):
"""Link table to back `OCIRecipe.processors`."""
diff --git a/lib/lp/oci/model/ocirecipebuild.py b/lib/lp/oci/model/ocirecipebuild.py
index 21c1cce..54034e2 100644
--- a/lib/lp/oci/model/ocirecipebuild.py
+++ b/lib/lp/oci/model/ocirecipebuild.py
@@ -44,6 +44,10 @@ from lp.oci.interfaces.ocirecipebuild import (
IOCIRecipeBuild,
IOCIRecipeBuildSet,
)
+from lp.oci.model.ocirecipebuildjob import (
+ OCIRecipeBuildJob,
+ OCIRecipeBuildJobType,
+ )
from lp.registry.interfaces.pocket import PackagePublishingPocket
from lp.registry.model.person import Person
from lp.services.config import config
@@ -55,6 +59,11 @@ from lp.services.database.interfaces import (
IMasterStore,
IStore,
)
+<<<<<<< lib/lp/oci/model/ocirecipebuild.py
+=======
+from lp.services.features import getFeatureFlag
+from lp.services.job.model.job import Job
+>>>>>>> lib/lp/oci/model/ocirecipebuild.py
from lp.services.librarian.model import (
LibraryFileAlias,
LibraryFileContent,
@@ -365,6 +374,21 @@ class OCIRecipeBuild(PackageBuildMixin, Storm):
and self.digests is not None)
return layer_files_present and metadata_present
+ @property
+ def registry_upload_jobs(self):
+ jobs = Store.of(self).find(
+ OCIRecipeBuildJob,
+ OCIRecipeBuildJob.build == self,
+ OCIRecipeBuildJob.job_type == OCIRecipeBuildJobType.REGISTRY_UPLOAD
+ )
+ jobs.order_by(Desc(OCIRecipeBuildJob.job_id))
+
+ def preload_jobs(rows):
+ load_related(Job, rows, ["job_id"])
+
+ return DecoratedResultSet(
+ jobs, lambda job: job.makeDerived(), pre_iter_hook=preload_jobs)
+
@implementer(IOCIRecipeBuildSet)
class OCIRecipeBuildSet(SpecificBuildFarmJobSourceMixin):
diff --git a/lib/lp/oci/model/ocirecipebuildjob.py b/lib/lp/oci/model/ocirecipebuildjob.py
index 7d8699c..a93f7fe 100644
--- a/lib/lp/oci/model/ocirecipebuildjob.py
+++ b/lib/lp/oci/model/ocirecipebuildjob.py
@@ -11,6 +11,7 @@ __all__ = [
'OCIRecipeBuildJobType',
]
+
from lazr.delegates import delegate_to
from lazr.enum import (
DBEnumeratedType,
@@ -21,10 +22,21 @@ from storm.locals import (
Int,
Reference,
)
-from zope.interface import implementer
+from zope.component.interfaces import ObjectEvent
+from zope.event import notify
+from zope.interface import (
+ implementer,
+ provider,
+ )
from lp.app.errors import NotFoundError
-from lp.oci.interfaces.ocirecipebuildjob import IOCIRecipeBuildJob
+from lp.oci.interfaces.ocirecipebuildjob import (
+ IOCIRecipeBuildJob,
+ IOCIRecipeBuildRegistryUploadStatusChangedEvent,
+ IOCIRegistryUploadJob,
+ IOCIRegistryUploadJobSource,
+ )
+from lp.oci.model.ociregistryclient import OCIRegistryClient
from lp.services.database.enumcol import DBEnum
from lp.services.database.interfaces import IStore
from lp.services.database.stormbase import StormBase
@@ -33,14 +45,12 @@ from lp.services.job.model.job import (
Job,
)
from lp.services.job.runner import BaseRunnableJob
+from lp.services.propertycache import get_property_cache
class OCIRecipeBuildJobType(DBEnumeratedType):
"""Values that `OCIBuildJobType.job_type` can take."""
- # XXX twom (2020-04-02) This does not currently have a concrete
- # implementation, awaiting registry upload.
-
REGISTRY_UPLOAD = DBItem(0, """
Registry upload
@@ -48,6 +58,12 @@ class OCIRecipeBuildJobType(DBEnumeratedType):
""")
+@implementer(IOCIRecipeBuildRegistryUploadStatusChangedEvent)
+class OCIRecipeBuildegistryUploadStatusChangedEvent(ObjectEvent):
+ """See `IOCIRecipeBuildRegistryUploadStatusChangedEvent`."""
+
+
+
@implementer(IOCIRecipeBuildJob)
class OCIRecipeBuildJob(StormBase):
"""See `IOCIRecipeBuildJob`."""
@@ -82,7 +98,7 @@ class OCIRecipeBuildJob(StormBase):
self.json_data = json_data
def makeDerived(self):
- return OCIRecipeBuildJob.makeSubclass(self)
+ return OCIRecipeBuildJobDerived.makeSubclass(self)
@delegate_to(IOCIRecipeBuildJob)
@@ -138,3 +154,35 @@ class OCIRecipeBuildJobDerived(BaseRunnableJob):
('oci_project_name', self.context.build.recipe.oci_project.name)
])
return oops_vars
+
+
+@implementer(IOCIRegistryUploadJob)
+@provider(IOCIRegistryUploadJobSource)
+class OCIRegistryUploadJob(OCIRecipeBuildJobDerived):
+
+ class_job_type = OCIRecipeBuildJobType.REGISTRY_UPLOAD
+
+ @classmethod
+ def create(cls, build):
+ """See `IOCIRegistryUploadJobSource`"""
+ oci_build_job = OCIRecipeBuildJob(
+ build, cls.class_job_type, {})
+ job = cls(oci_build_job)
+ job.celeryRunOnCommit()
+ del get_property_cache(build).last_registry_upload_job
+ notify(OCIRecipeBuildegistryUploadStatusChangedEvent(build))
+ return job
+
+ @property
+ def error_message(self):
+ """See `IOCIRegistryUploadJob`."""
+ return self.json_data.get("error_message")
+
+ @error_message.setter
+ def error_message(self, message):
+ """See `IOCIRegistryUploadJob`."""
+ self.json_data["error_message"] = message
+
+ def run(self):
+ """See `IRunnableJob`."""
+ OCIRegistryClient.upload(self.build)
diff --git a/lib/lp/oci/model/ociregistryclient.py b/lib/lp/oci/model/ociregistryclient.py
new file mode 100644
index 0000000..2f892b5
--- /dev/null
+++ b/lib/lp/oci/model/ociregistryclient.py
@@ -0,0 +1,264 @@
+# Copyright 2020 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Client for talking to an OCI registry."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+ 'OCIRegistryClient'
+]
+
+
+from io import BytesIO
+import hashlib
+import json
+import logging
+import tarfile
+
+from requests.exceptions import HTTPError
+from zope.interface import implementer
+
+from lp.oci.interfaces.ociregistryclient import (
+ BlobUploadFailed,
+ IOCIRegistryClient,
+ ManifestUploadFailed,
+ )
+from lp.services.timeout import urlfetch
+
+log = logging.getLogger("ociregistryclient")
+
+
+@implementer(IOCIRegistryClient)
+class OCIRegistryClient:
+
+ @classmethod
+ def _getJSONfile(cls, reference):
+ """Read JSON out of a `LibraryFileAlias`."""
+ _, lfa, lfc = reference
+ try:
+ lfa.open()
+ return json.loads(lfa.read())
+ finally:
+ lfa.close()
+
+ @classmethod
+ def _upload(cls, digest, push_rule, name, fileobj):
+ """Upload a blob to the registry, using a given digest.
+
+ :param digest: The digest to store the file under.
+ :param push_rule: `OCIPushRule` to use for the URL and credentials.
+ :param name: Name of the image the blob is part of.
+ :param fileobj: An object that looks like a buffer.
+
+ :raises BlobUploadFailed: if the registry does not accept the blob.
+ """
+ # Check if it already exists
+ try:
+ head_response = urlfetch(
+ "{}/v2/{}/blobs/{}".format(
+ push_rule.registry_credentials.url, name, digest),
+ method="HEAD")
+ if head_response.status_code == 200:
+ log.info("{} already found".format(digest))
+ return
+ except HTTPError as http_error:
+ # A 404 is fine, we're about to upload the layer anyway
+ if http_error.response.status_code != 404:
+ raise http_error
+
+ post_response = urlfetch(
+ "{}/v2/{}/blobs/uploads/".format(
+ push_rule.registry_credentials.url, name),
+ method="POST")
+
+ post_location = post_response.headers["Location"]
+ query_parsed = {"digest": digest}
+
+ put_response = urlfetch(
+ post_location,
+ params=query_parsed,
+ data=fileobj,
+ method="PUT")
+
+ if put_response.status_code != 201:
+ raise BlobUploadFailed(
+ "Upload of {} for {} failed".format(digest, name))
+
+ @classmethod
+ def _upload_layer(cls, digest, push_rule, name, lfa):
+ """Upload a layer blob to the registry.
+
+ Uses _upload, but opens the LFA and extracts the necessary files
+ from the .tar.gz first.
+
+ :param digest: The digest to store the file under.
+ :param push_rule: `OCIPushRule` to use for the URL and credentials.
+ :param name: Name of the image the blob is part of.
+ :param lfa: The `LibraryFileAlias` for the layer.
+ """
+ lfa.open()
+ try:
+ un_zipped = tarfile.open(fileobj=lfa, mode='r|gz')
+ for tarinfo in un_zipped:
+ if tarinfo.name != 'layer.tar':
+ continue
+ fileobj = un_zipped.extractfile(tarinfo)
+ cls._upload(digest, push_rule, name, fileobj)
+ finally:
+ lfa.close()
+
+ @classmethod
+ def _build_registry_manifest(cls, digests,
+ config, config_json, config_sha):
+ """Create an image manifest for the uploading image.
+
+ This involves nearly everything as digests and lengths are required.
+ This method creates a minimal manifest, some fields are missing.
+
+ :param digests: Dict of the various digests involved.
+ :param config: The contents of the manifest config file as a dict.
+ :param config_json: The config file as a JSON string.
+ :param config_sha: The sha256sum of the config JSON string.
+ """
+
+ # Create the initial manifest data with empty layer information
+ manifest = {
+ "schemaVersion": 2,
+ "mediaType":
+ "application/vnd.docker.distribution.manifest.v2+json",
+ "config": {
+ "mediaType": "application/vnd.docker.container.image.v1+json",
+ "size": len(config_json),
+ "digest": "sha256:{}".format(config_sha),
+ },
+ "layers": []}
+
+ # Fill in the layer information
+ for layer in config["rootfs"]["diff_ids"]:
+ manifest["layers"].append({
+ "mediaType":
+ "application/vnd.docker.image.rootfs.diff.tar.gzip",
+ "size": 0, # XXX twom 2020-04-14 We can get this from LFA
+ "digest": layer})
+ return manifest
+
+ @classmethod
+ def _preloadFiles(cls, build, manifest, digests):
+ """Preload the data from the librarian to avoid multiple fetches
+ if there is more than one push rule for a build.
+
+ :param build: The referencing `OCIRecipeBuild`.
+ :param manifest: The manifest from the built image.
+ :param digests: Dict of the various digests involved.
+ """
+ data = {}
+ for section in manifest:
+ # Load the matching config file for this section
+ config = cls._getJSONfile(
+ build.getByFileName(section['Config']))
+ files = {"config_file": config}
+ for diff_id in config["rootfs"]["diff_ids"]:
+ # We may have already seen this diff ID.
+ if files.get(diff_id):
+ continue
+ # Retrieve the layer files.
+ # This doesn't read the content, so there is potential
+ # for multiple fetches, but the files can be arbitary size
+ # Potentially gigabytes.
+ files[diff_id] = {}
+ source_digest = digests[diff_id]["digest"]
+ _, lfa, _ = build.getLayerFileByDigest(source_digest)
+ files[diff_id] = lfa
+ data[section["Config"]] = files
+ return data
+
+ @classmethod
+ def _calculateName(cls, build, push_rule):
+ """Work out what the name for the image should be.
+
+ :param build: `OCIRecipeBuild` representing this build.
+ :param push_rule: `OCIPushRule` that we are using.
+ """
+ return "{}/{}".format(
+ build.recipe.oci_project.pillar.name,
+ push_rule.image_name)
+
+ @classmethod
+ def _calculateTag(cls, build, push_rule):
+ """Work out the base tag for the image should be.
+
+ :param build: `OCIRecipeBuild` representing this build.
+ :param push_rule: `OCIPushRule` that we are using.
+ """
+ # XXX twom 2020-04-17 This needs to include OCIProjectSeries and
+ # base image name
+
+ return "{}".format("edge")
+
+ @classmethod
+ def upload(cls, build):
+ """Upload the artifacts from an OCIRecipeBuild to a registry.
+
+ :param build: `OCIRecipeBuild` representing this build.
+ :raises ManifestUploadFailed: If the final registry manifest fails to
+ upload due to network or validity.
+ """
+ # Get the required metadata files
+ manifest = cls._getJSONfile(build.manifest)
+ digests_list = cls._getJSONfile(build.digests)
+ digests = {}
+ for digest_dict in digests_list:
+ for k, v in digest_dict.items():
+ digests[k] = v
+
+ # Preload the requested files
+ preloaded_data = cls._preloadFiles(build, manifest, digests)
+
+ for push_rule in build.recipe.push_rules:
+ for section in manifest:
+ # Work out names and tags
+ image_name = cls._calculateName(build, push_rule)
+ tag = cls._calculateTag(build, push_rule)
+ file_data = preloaded_data[section["Config"]]
+ config = file_data["config_file"]
+ # Upload the layers involved
+ for diff_id in config["rootfs"]["diff_ids"]:
+ cls._upload_layer(
+ diff_id,
+ push_rule,
+ image_name,
+ file_data[diff_id])
+ # The config file is required in different forms, so we can
+ # calculate the sha, work these out and upload
+ config_json = json.dumps(config).encode()
+ config_sha = hashlib.sha256(config_json).hexdigest()
+ cls._upload(
+ "sha256:{}".format(config_sha),
+ push_rule,
+ image_name,
+ BytesIO(config_json))
+
+ # Build the registry manifest from the image manifest
+ # and associated configs
+ registry_manifest = cls._build_registry_manifest(
+ digests, config, config_json, config_sha)
+
+ # Upload the registry manifest
+ manifest_response = urlfetch(
+ "{}/v2/{}/manifests/{}".format(
+ push_rule.registry_credentials.url,
+ image_name,
+ tag),
+ json=registry_manifest,
+ headers={
+ "Content-Type":
+ "application/"
+ "vnd.docker.distribution.manifest.v2+json"
+ },
+ method="PUT")
+ if manifest_response.status_code != 201:
+ raise ManifestUploadFailed(
+ "Failed to upload manifest for {} in {}".format(
+ build.recipe.name, build.id))
diff --git a/lib/lp/oci/model/ociregistrycredentials.py b/lib/lp/oci/model/ociregistrycredentials.py
index 2a70159..03b0f72 100644
--- a/lib/lp/oci/model/ociregistrycredentials.py
+++ b/lib/lp/oci/model/ociregistrycredentials.py
@@ -80,8 +80,8 @@ class OCIRegistryCredentials(Storm):
def getCredentials(self):
container = getUtility(IEncryptedContainer, "oci-registry-secrets")
try:
- return json.loads(container.decrypt((
- self._credentials['credentials_encrypted'])).decode("UTF-8"))
+ return json.loads(container.decrypt(
+ self._credentials['credentials_encrypted']).decode("UTF-8"))
except CryptoError as e:
# XXX twom 2020-03-18 This needs a better error
# see SnapStoreClient.UnauthorizedUploadResponse
diff --git a/lib/lp/oci/subscribers/ocirecipebuild.py b/lib/lp/oci/subscribers/ocirecipebuild.py
index 3e7f80d..4703e3e 100644
--- a/lib/lp/oci/subscribers/ocirecipebuild.py
+++ b/lib/lp/oci/subscribers/ocirecipebuild.py
@@ -9,8 +9,10 @@ __metaclass__ = type
from zope.component import getUtility
+from lp.buildmaster.enums import BuildStatus
from lp.oci.interfaces.ocirecipe import OCI_RECIPE_WEBHOOKS_FEATURE_FLAG
from lp.oci.interfaces.ocirecipebuild import IOCIRecipeBuild
+from lp.oci.interfaces.ocirecipebuildjob import IOCIRegistryUploadJobSource
from lp.services.features import getFeatureFlag
from lp.services.webapp.publisher import canonical_url
from lp.services.webhooks.interfaces import IWebhookSet
@@ -40,3 +42,6 @@ def oci_recipe_build_status_changed(build, event):
if event.edited_fields is not None:
if "status" in event.edited_fields:
_trigger_oci_recipe_build_webhook(build, "status-changed")
+ if (build.recipe.can_upload_to_registry and
+ build.status == BuildStatus.FULLYBUILT):
+ getUtility(IOCIRegistryUploadJobSource).create(build)
diff --git a/lib/lp/oci/tests/test_ocirecipebuildjob.py b/lib/lp/oci/tests/test_ocirecipebuildjob.py
index cf85684..158892b 100644
--- a/lib/lp/oci/tests/test_ocirecipebuildjob.py
+++ b/lib/lp/oci/tests/test_ocirecipebuildjob.py
@@ -7,16 +7,25 @@ from __future__ import absolute_import, print_function, unicode_literals
__metaclass__ = type
+from fixtures import FakeLogger
+
from lp.oci.interfaces.ocirecipe import OCI_RECIPE_ALLOW_CREATE
-from lp.oci.interfaces.ocirecipebuildjob import IOCIRecipeBuildJob
+from lp.oci.interfaces.ocirecipebuildjob import (
+ IOCIRecipeBuildJob,
+ IOCIRegistryUploadJob,
+ )
from lp.oci.model.ocirecipebuildjob import (
OCIRecipeBuildJob,
OCIRecipeBuildJobDerived,
OCIRecipeBuildJobType,
+ OCIRegistryUploadJob,
)
from lp.services.features.testing import FeatureFixture
from lp.testing import TestCaseWithFactory
-from lp.testing.layers import DatabaseFunctionalLayer
+from lp.testing.layers import (
+ DatabaseFunctionalLayer,
+ LaunchpadZopelessLayer,
+ )
class FakeOCIBuildJob(OCIRecipeBuildJobDerived):
@@ -52,3 +61,24 @@ class TestOCIRecipeBuildJob(TestCaseWithFactory):
('oci_project_name', oci_build.recipe.oci_project.name),
]
self.assertEqual(expected, oops)
+
+
+class TestOCIRegistryUploadJobJob(TestCaseWithFactory):
+
+ layer = LaunchpadZopelessLayer
+
+ def setUp(self):
+ super(TestOCIRegistryUploadJobJob, self).setUp()
+ self.useFixture(FeatureFixture({OCI_RECIPE_ALLOW_CREATE: 'on'}))
+
+ def test_provides_interface(self):
+ # `OCIRegistryUploadJob` objects provide `IOCIRegistryUploadJob`.
+ ocibuild = self.factory.makeOCIRecipeBuild()
+ job = OCIRegistryUploadJob.create(ocibuild)
+ self.assertProvides(job, IOCIRegistryUploadJob)
+
+ def test_run(self):
+ logger = self.useFixture(FakeLogger())
+ ocibuild = self.factory.makeOCIRecipeBuild()
+ self.assertContentEqual([], ocibuild.registry_upload_jobs)
+ job = OCIRegistryUploadJob.create(ocibuild)
diff --git a/lib/lp/oci/tests/test_ociregistryclient.py b/lib/lp/oci/tests/test_ociregistryclient.py
new file mode 100644
index 0000000..9345ec7
--- /dev/null
+++ b/lib/lp/oci/tests/test_ociregistryclient.py
@@ -0,0 +1,206 @@
+# Copyright 2020 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Tests for the OCI Registry client."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+
+import json
+
+from fixtures import MockPatch
+import responses
+import transaction
+
+from lp.oci.model.ociregistryclient import OCIRegistryClient
+from lp.oci.tests.helpers import OCIConfigHelperMixin
+
+from lp.testing import TestCaseWithFactory
+from lp.testing.layers import LaunchpadZopelessLayer
+from testtools.matchers import Equals, MatchesDict, MatchesListwise
+from requests.exceptions import HTTPError
+
+
+class TestOCIRegistryClient(OCIConfigHelperMixin, TestCaseWithFactory):
+
+ layer = LaunchpadZopelessLayer
+
+ def setUp(self):
+ super(TestOCIRegistryClient, self).setUp()
+ self.setConfig()
+ self.manifest = [{
+ "Config": "config_file_1.json",
+ "Layers": ["layer_1/layer.tar", "layer_2/layer.tar"]}]
+ self.digests = [{
+ "diff_id_1": {
+ "digest": "digest_1",
+ "source": "test/base_1",
+ "layer_id": "layer_1"
+ },
+ "diff_id_2": {
+ "digest": "digest_2",
+ "source": "",
+ "layer_id": "layer_2"
+ }
+ }]
+ self.config = {"rootfs": {"diff_ids": ["diff_id_1", "diff_id_2"]}}
+ self.build = self.factory.makeOCIRecipeBuild()
+ self.factory.makeOCIPushRule(recipe=self.build.recipe)
+ self.client = OCIRegistryClient()
+
+ def _makeFiles(self):
+ self.factory.makeOCIFile(
+ build=self.build,
+ content=json.dumps(self.manifest),
+ filename='manifest.json',
+ )
+ self.factory.makeOCIFile(
+ build=self.build,
+ content=json.dumps(self.digests),
+ filename='digests.json',
+ )
+ self.factory.makeOCIFile(
+ build=self.build,
+ content=json.dumps(self.config),
+ filename='config_file_1.json'
+ )
+
+ # make layer files
+ self.layer_1_file = self.factory.makeOCIFile(
+ build=self.build,
+ content="digest_1",
+ filename="digest_1_filename",
+ layer_file_digest="digest_1"
+ )
+ self.layer_2_file = self.factory.makeOCIFile(
+ build=self.build,
+ content="digest_2",
+ filename="digest_2_filename",
+ layer_file_digest="digest_2"
+ )
+
+ transaction.commit()
+
+ @responses.activate
+ def test_upload(self):
+ self._makeFiles()
+ self.useFixture(MockPatch(
+ "lp.oci.model.ociregistryclient.OCIRegistryClient._upload"))
+ self.useFixture(MockPatch(
+ "lp.oci.model.ociregistryclient.OCIRegistryClient._upload_layer"))
+
+ manifests_url = "{}/v2/{}/{}/manifests/edge".format(
+ self.build.recipe.push_rules[0].registry_credentials.url,
+ self.build.recipe.oci_project.pillar.name,
+ self.build.recipe.push_rules[0].image_name
+ )
+ responses.add("PUT", manifests_url, status=201)
+ self.client.upload(self.build)
+
+ request = json.loads(responses.calls[0].request.body)
+
+ self.assertThat(request, MatchesDict({
+ "layers": MatchesListwise([
+ MatchesDict({
+ "mediaType": Equals(
+ "application/vnd.docker.image.rootfs.diff.tar.gzip"),
+ "digest": Equals("diff_id_1"),
+ "size": Equals(0)}),
+ MatchesDict({
+ "mediaType": Equals(
+ "application/vnd.docker.image.rootfs.diff.tar.gzip"),
+ "digest": Equals("diff_id_2"),
+ "size": Equals(0)})
+ ]),
+ "schemaVersion": Equals(2),
+ "config": MatchesDict({
+ "mediaType": Equals(
+ "application/vnd.docker.container.image.v1+json"),
+ "digest": Equals(
+ "sha256:33b69b4b6e106f9fc7a8b93409"
+ "36c85cf7f84b2d017e7b55bee6ab214761f6ab"),
+ "size": Equals(52)
+ }),
+ "mediaType": Equals(
+ "application/vnd.docker.distribution.manifest.v2+json")
+ }))
+
+ def test_preloadFiles(self):
+ self._makeFiles()
+ files = self.client._preloadFiles(
+ self.build, self.manifest, self.digests[0])
+
+ self.assertThat(files, MatchesDict({
+ 'config_file_1.json': MatchesDict({
+ 'config_file': Equals(self.config),
+ 'diff_id_1': Equals(self.layer_1_file.library_file),
+ 'diff_id_2': Equals(self.layer_2_file.library_file)})}))
+
+ def test_calculateTag(self):
+ result = self.client._calculateTag(
+ self.build, self.build.recipe.push_rules[0])
+ self.assertEqual("edge", result)
+
+ def test_calculateName(self):
+ result = self.client._calculateName(
+ self.build, self.build.recipe.push_rules[0])
+ expected = "{}/{}".format(
+ self.build.recipe.oci_project.pillar.name,
+ self.build.recipe.push_rules[0].image_name)
+ self.assertEqual(expected, result)
+
+ def test_build_registry_manifest(self):
+ manifest = self.client._build_registry_manifest(
+ self.digests[0],
+ self.config,
+ json.dumps(self.config),
+ "config-sha")
+ self.assertThat(manifest, MatchesDict({
+ "layers": MatchesListwise([
+ MatchesDict({
+ "mediaType": Equals(
+ "application/vnd.docker.image.rootfs.diff.tar.gzip"),
+ "digest": Equals("diff_id_1"),
+ "size": Equals(0)}),
+ MatchesDict({
+ "mediaType": Equals(
+ "application/vnd.docker.image.rootfs.diff.tar.gzip"),
+ "digest": Equals("diff_id_2"),
+ "size": Equals(0)})
+ ]),
+ "schemaVersion": Equals(2),
+ "config": MatchesDict({
+ "mediaType": Equals(
+ "application/vnd.docker.container.image.v1+json"),
+ "digest": Equals("sha256:config-sha"),
+ "size": Equals(52)
+ }),
+ "mediaType": Equals(
+ "application/vnd.docker.distribution.manifest.v2+json")
+ }))
+
+ @responses.activate
+ def test_upload_handles_existing(self):
+ blobs_url = "{}/v2/{}/blobs/{}".format(
+ self.build.recipe.push_rules[0].registry_credentials.url,
+ "test-name",
+ "test-digest")
+ responses.add("HEAD", blobs_url, status=200)
+ self.client._upload(
+ "test-digest", self.build.recipe.push_rules[0], "test-name", None)
+
+ @responses.activate
+ def test_upload_raises_non_404(self):
+ blobs_url = "{}/v2/{}/blobs/{}".format(
+ self.build.recipe.push_rules[0].registry_credentials.url,
+ "test-name",
+ "test-digest")
+ responses.add("HEAD", blobs_url, status=500)
+ self.assertRaises(
+ HTTPError,
+ self.client._upload,
+ "test-digest",
+ self.build.recipe.push_rules[0],
+ "test-name",
+ None)
diff --git a/lib/lp/services/config/schema-lazr.conf b/lib/lp/services/config/schema-lazr.conf
index c43a908..b8f1c33 100644
--- a/lib/lp/services/config/schema-lazr.conf
+++ b/lib/lp/services/config/schema-lazr.conf
@@ -1815,6 +1815,7 @@ job_sources:
IExpiringMembershipNotificationJobSource,
IGitRepositoryModifiedMailJobSource,
IMembershipNotificationJobSource,
+ IOCIRegistryUploadJobSource,
IPackageUploadNotificationJobSource,
IPersonDeactivateJobSource,
IPersonMergeJobSource,
@@ -1978,6 +1979,11 @@ module: lp.snappy.interfaces.snapbuildjob
dbuser: snap-build-job
crontab_group: MAIN
+[IOCIRegistryUploadJobSource]
+module: lp.oci.interfaces.ocirecipebuildjob
+dbuser: oci-build-job
+crontab_group: MAIN
+
[ITeamInvitationNotificationJobSource]
module: lp.registry.interfaces.persontransferjob
dbuser: person-transfer-job
