launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #27449
[Merge] ~cjwatson/launchpad:charmhub-client-push-release into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:charmhub-client-push-release into launchpad:master.
Commit message:
Extend Charmhub client to handle pushing and releasing charms
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/407380
I've only checked this against the publishergw API by eye. A later branch will add an upload job, at which point we'll be able to do end-to-end testing.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:charmhub-client-push-release into launchpad:master.
diff --git a/lib/lp/charms/interfaces/charmhubclient.py b/lib/lp/charms/interfaces/charmhubclient.py
index 8beb251..4e9a528 100644
--- a/lib/lp/charms/interfaces/charmhubclient.py
+++ b/lib/lp/charms/interfaces/charmhubclient.py
@@ -6,7 +6,13 @@
__all__ = [
"BadExchangeMacaroonsResponse",
"BadRequestPackageUploadResponse",
+ "BadReviewStatusResponse",
"ICharmhubClient",
+ "ReleaseFailedResponse",
+ "ReviewFailedResponse",
+ "UnauthorizedUploadResponse",
+ "UploadFailedResponse",
+ "UploadNotReviewedYetResponse",
]
import http.client
@@ -33,6 +39,30 @@ class BadExchangeMacaroonsResponse(CharmhubError):
pass
+class UploadFailedResponse(CharmhubError):
+ pass
+
+
+class UnauthorizedUploadResponse(CharmhubError):
+ pass
+
+
+class BadReviewStatusResponse(CharmhubError):
+ pass
+
+
+class UploadNotReviewedYetResponse(CharmhubError):
+ pass
+
+
+class ReviewFailedResponse(CharmhubError):
+ pass
+
+
+class ReleaseFailedResponse(CharmhubError):
+ pass
+
+
class ICharmhubClient(Interface):
"""Interface for the API provided by Charmhub."""
@@ -61,3 +91,36 @@ class ICharmhubClient(Interface):
:return: A serialized macaroon from Charmhub with no third-party
Candid caveat.
"""
+
+ def upload(build):
+ """Upload a charm recipe build to CharmHub.
+
+ :param build: The `ICharmRecipeBuild` to upload.
+ :return: A URL to poll for upload processing status.
+ :raises UnauthorizedUploadResponse: if the user who authorised this
+ upload is not themselves authorised to upload the snap in
+ question.
+ :raises UploadFailedResponse: if uploading the build to Charmhub
+ failed.
+ """
+
+ def checkStatus(status_url):
+ """Poll Charmhub once for upload scan status.
+
+ :param status_url: A URL as returned by `upload`.
+ :raises UploadNotReviewedYetResponse: if the upload has not yet been
+ reviewed.
+ :raises BadReviewStatusResponse: if Charmhub failed to review the
+ upload.
+ :return: The Charmhub revision number for the upload.
+ """
+
+ def release(build, revision):
+ """Tell Charmhub to release a build to specified channels.
+
+ :param build: The `ICharmRecipeBuild` to release.
+ :param revision: The revision returned by Charmhub when uploading
+ the build.
+ :raises ReleaseFailedResponse: if Charmhub failed to release the
+ build.
+ """
diff --git a/lib/lp/charms/interfaces/charmrecipe.py b/lib/lp/charms/interfaces/charmrecipe.py
index c6dac7d..46195b9 100644
--- a/lib/lp/charms/interfaces/charmrecipe.py
+++ b/lib/lp/charms/interfaces/charmrecipe.py
@@ -274,6 +274,12 @@ class ICharmRecipeView(Interface):
title=_("Private"), required=False, readonly=False,
description=_("Whether this charm recipe is private."))
+ can_upload_to_store = Bool(
+ title=_("Can upload to Charmhub"), required=True, readonly=True,
+ description=_(
+ "Whether everything is set up to allow uploading builds of this "
+ "charm recipe to Charmhub."))
+
def getAllowedInformationTypes(user):
"""Get a list of acceptable `InformationType`s for this charm recipe.
diff --git a/lib/lp/charms/model/charmhubclient.py b/lib/lp/charms/model/charmhubclient.py
index be195d3..9b2627f 100644
--- a/lib/lp/charms/model/charmhubclient.py
+++ b/lib/lp/charms/model/charmhubclient.py
@@ -8,24 +8,53 @@ __all__ = [
]
from base64 import b64encode
+from urllib.parse import quote
from lazr.restful.utils import get_current_browser_request
from pymacaroons import Macaroon
from pymacaroons.serializers import JsonSerializer
import requests
+from requests_toolbelt import MultipartEncoder
+from zope.component import getUtility
from zope.interface import implementer
from lp.charms.interfaces.charmhubclient import (
BadExchangeMacaroonsResponse,
BadRequestPackageUploadResponse,
+ BadReviewStatusResponse,
ICharmhubClient,
+ ReleaseFailedResponse,
+ ReviewFailedResponse,
+ UnauthorizedUploadResponse,
+ UploadFailedResponse,
+ UploadNotReviewedYetResponse,
)
from lp.services.config import config
+from lp.services.crypto.interfaces import (
+ CryptoError,
+ IEncryptedContainer,
+ )
+from lp.services.librarian.utils import EncodableLibraryFileAlias
from lp.services.timeline.requesttimeline import get_request_timeline
from lp.services.timeout import urlfetch
from lp.services.webapp.url import urlappend
+def _get_macaroon(recipe):
+ """Get the Charmhub macaroon for a recipe."""
+ store_secrets = recipe.store_secrets or {}
+ macaroon_raw = store_secrets.get("exchanged_encrypted")
+ if macaroon_raw is None:
+ raise UnauthorizedUploadResponse(
+ "{} is not authorized for upload to Charmhub".format(recipe))
+ container = getUtility(IEncryptedContainer, "charmhub-secrets")
+ try:
+ return container.decrypt(macaroon_raw).decode()
+ except CryptoError as e:
+ raise UnauthorizedUploadResponse(
+ "Failed to decrypt macaroon: {}".format(e))
+
+
@implementer(ICharmhubClient)
class CharmhubClient:
"""A client for the API provided by Charmhub."""
@@ -48,10 +77,10 @@ class CharmhubClient:
except ValueError:
pass
else:
- if "error_list" in response_data:
+ if "error-list" in response_data:
error_message = "\n".join(
error["message"]
- for error in response_data["error_list"])
+ for error in response_data["error-list"])
detail = requests_error.response.content.decode(errors="replace")
can_retry = requests_error.response.status_code in (502, 503)
return error_class(error_message, detail=detail, can_retry=can_retry)
@@ -115,3 +144,136 @@ class CharmhubClient:
raise cls._makeCharmhubError(BadExchangeMacaroonsResponse, e)
finally:
timeline_action.finish()
+
+ @classmethod
+ def _uploadToStorage(cls, lfa):
+ """Upload a single file to Charmhub's storage."""
+ assert config.charms.charmhub_storage_url is not None
+ unscanned_upload_url = urlappend(
+ config.charms.charmhub_storage_url, "unscanned-upload/")
+ lfa.open()
+ try:
+ lfa_wrapper = EncodableLibraryFileAlias(lfa)
+ encoder = MultipartEncoder(
+ fields={
+ "binary": (
+ lfa.filename, lfa_wrapper, "application/octet-stream"),
+ })
+ request = get_current_browser_request()
+ timeline_action = get_request_timeline(request).start(
+ "charm-storage-push", lfa.filename, allow_nested=True)
+ try:
+ response = urlfetch(
+ unscanned_upload_url, method="POST", data=encoder,
+ headers={
+ "Content-Type": encoder.content_type,
+ "Accept": "application/json",
+ })
+ response_data = response.json()
+ if not response_data.get("successful", False):
+ raise UploadFailedResponse(response.text)
+ return response_data["upload_id"]
+ except requests.HTTPError as e:
+ raise cls._makeCharmhubError(UploadFailedResponse, e)
+ finally:
+ timeline_action.finish()
+ finally:
+ lfa.close()
+
+ @classmethod
+ def _push(cls, build, upload_id):
+ """Push an already-uploaded charm to Charmhub."""
+ recipe = build.recipe
+ assert config.charms.charmhub_url is not None
+ assert recipe.store_name is not None
+ assert recipe.store_secrets is not None
+ push_url = urlappend(
+ config.charms.charmhub_url,
+ "v1/charm/{}/revisions".format(quote(recipe.store_name)))
+ macaroon_raw = _get_macaroon(recipe)
+ data = {"upload-id": upload_id}
+ request = get_current_browser_request()
+ timeline_action = get_request_timeline(request).start(
+ "charm-push", recipe.store_name, allow_nested=True)
+ try:
+ response = urlfetch(
+ push_url, method="POST",
+ headers={"Authorization": "Macaroon {}".format(macaroon_raw)},
+ json=data)
+ response_data = response.json()
+ return response_data["status-url"]
+ except requests.HTTPError as e:
+ if e.response.status_code == 401:
+ raise cls._makeCharmhubError(UnauthorizedUploadResponse, e)
+ else:
+ raise cls._makeCharmhubError(UploadFailedResponse, e)
+ finally:
+ timeline_action.finish()
+
+ @classmethod
+ def upload(cls, build):
+ """See `ICharmhubClient`."""
+ assert build.recipe.can_upload_to_store
+ for _, lfa, _ in build.getFiles():
+ if not lfa.filename.endswith(".charm"):
+ continue
+ upload_id = cls._uploadToStorage(lfa)
+ return cls._push(build, upload_id)
+
+ @classmethod
+ def checkStatus(cls, build, status_url):
+ """See `ICharmhubClient`."""
+ macaroon_raw = _get_macaroon(build.recipe)
+ request = get_current_browser_request()
+ timeline_action = get_request_timeline(request).start(
+ "charm-check-status", status_url, allow_nested=True)
+ try:
+ response = urlfetch(
+ status_url,
+ headers={"Authorization": "Macaroon {}".format(macaroon_raw)})
+ response_data = response.json()
+ # We're asking for a single upload ID, so the response should
+ # only have one revision.
+ if len(response_data.get("revisions", [])) != 1:
+ raise BadReviewStatusResponse(response.text)
+ [revision] = response_data["revisions"]
+ if revision["status"] == "approved":
+ return revision["revision"]
+ elif revision["status"] == "rejected":
+ error_message = "\n".join(
+ error["message"] for error in revision["errors"])
+ raise ReviewFailedResponse(error_message)
+ else:
+ raise UploadNotReviewedYetResponse()
+ except requests.HTTPError as e:
+ raise cls._makeCharmhubError(BadReviewStatusResponse, e)
+ finally:
+ timeline_action.finish()
+
+ @classmethod
+ def release(cls, build, revision):
+ """See `ICharmhubClient`."""
+ assert config.charms.charmhub_url is not None
+ recipe = build.recipe
+ assert recipe.store_name is not None
+ assert recipe.store_secrets is not None
+ assert recipe.store_channels
+ release_url = urlappend(
+ config.charms.charmhub_url,
+ "v1/charm/{}/releases".format(quote(recipe.store_name)))
+ macaroon_raw = _get_macaroon(recipe)
+ data = [
+ {"channel": channel, "revision": revision}
+ for channel in recipe.store_channels]
+ request = get_current_browser_request()
+ timeline_action = get_request_timeline(request).start(
+ "charm-release", recipe.store_name, allow_nested=True)
+ try:
+ urlfetch(
+ release_url, method="POST",
+ headers={"Authorization": "Macaroon {}".format(macaroon_raw)},
+ json=data)
+ except requests.HTTPError as e:
+ raise cls._makeCharmhubError(ReleaseFailedResponse, e)
+ finally:
+ timeline_action.finish()
diff --git a/lib/lp/charms/model/charmrecipe.py b/lib/lp/charms/model/charmrecipe.py
index a3ce3b3..1907347 100644
--- a/lib/lp/charms/model/charmrecipe.py
+++ b/lib/lp/charms/model/charmrecipe.py
@@ -692,6 +692,14 @@ class CharmRecipe(StormBase):
container.encrypt(exchanged_macaroon_raw.encode()))
self.store_secrets.pop("root", None)
+ @property
+ def can_upload_to_store(self):
+ return (
+ config.charms.charmhub_url is not None and
+ self.store_name is not None and
+ self.store_secrets is not None and
+ "exchanged_encrypted" in self.store_secrets)
+
def destroySelf(self):
"""See `ICharmRecipe`."""
store = IStore(self)
diff --git a/lib/lp/charms/tests/test_charmhubclient.py b/lib/lp/charms/tests/test_charmhubclient.py
index b041055..864c0d6 100644
--- a/lib/lp/charms/tests/test_charmhubclient.py
+++ b/lib/lp/charms/tests/test_charmhubclient.py
@@ -4,68 +4,168 @@
"""Tests for communication with Charmhub."""
import base64
+import hashlib
+import io
import json
+from urllib.parse import quote
from lazr.restful.utils import get_current_browser_request
-from pymacaroons import Macaroon
+import multipart
+from nacl.public import PrivateKey
+from pymacaroons import (
+ Macaroon,
+ Verifier,
+ )
from pymacaroons.serializers import JsonSerializer
import responses
from testtools.matchers import (
AfterPreprocessing,
ContainsDict,
Equals,
+ Is,
+ Matcher,
MatchesAll,
+ MatchesDict,
+ MatchesListwise,
MatchesStructure,
+ Mismatch,
)
+import transaction
from zope.component import getUtility
+from zope.security.proxy import removeSecurityProxy
+from lp.buildmaster.enums import BuildStatus
from lp.charms.interfaces.charmhubclient import (
BadExchangeMacaroonsResponse,
BadRequestPackageUploadResponse,
+ BadReviewStatusResponse,
ICharmhubClient,
+ ReleaseFailedResponse,
+ ReviewFailedResponse,
+ UnauthorizedUploadResponse,
+ UploadFailedResponse,
+ UploadNotReviewedYetResponse,
)
from lp.charms.interfaces.charmrecipe import CHARM_RECIPE_ALLOW_CREATE
+from lp.services.crypto.interfaces import IEncryptedContainer
from lp.services.features.testing import FeatureFixture
from lp.services.timeline.requesttimeline import get_request_timeline
from lp.testing import TestCaseWithFactory
-from lp.testing.layers import ZopelessDatabaseLayer
+from lp.testing.dbuser import dbuser
+from lp.testing.layers import LaunchpadZopelessLayer
+
+
+class MacaroonVerifies(Matcher):
+ """Matches if a serialized macaroon passes verification."""
+
+ def __init__(self, key):
+ self.key = key
+ def match(self, macaroon_raw):
+ macaroon = Macaroon.deserialize(macaroon_raw)
+ try:
+ Verifier().verify(macaroon, self.key)
+ except Exception as e:
+ return Mismatch("Macaroon does not verify: %s" % e)
-class RequestMatches(MatchesStructure):
+
+class RequestMatches(MatchesAll):
"""Matches a request with the specified attributes."""
- def __init__(self, macaroons=None, json_data=None, **kwargs):
+ def __init__(self, macaroons=None, auth=None, json_data=None,
+ file_data=None, **kwargs):
+ matchers = []
kwargs = dict(kwargs)
if macaroons is not None:
- headers_matcher = ContainsDict({
+ matchers.append(MatchesStructure(headers=ContainsDict({
"Macaroons": AfterPreprocessing(
lambda v: json.loads(
base64.b64decode(v.encode()).decode()),
Equals([json.loads(m) for m in macaroons])),
- })
- if kwargs.get("headers"):
- headers_matcher = MatchesAll(
- kwargs["headers"], headers_matcher)
- kwargs["headers"] = headers_matcher
+ })))
+ if auth is not None:
+ auth_scheme, auth_params_matcher = auth
+ matchers.append(MatchesStructure(headers=ContainsDict({
+ "Authorization": AfterPreprocessing(
+ lambda v: v.split(" ", 1),
+ MatchesListwise([
+ Equals(auth_scheme),
+ auth_params_matcher,
+ ])),
+ })))
if json_data is not None:
- body_matcher = AfterPreprocessing(
- lambda b: json.loads(b.decode()), Equals(json_data))
- if kwargs.get("body"):
- body_matcher = MatchesAll(kwargs["body"], body_matcher)
- kwargs["body"] = body_matcher
- super().__init__(**kwargs)
+ matchers.append(MatchesStructure(body=AfterPreprocessing(
+ lambda b: json.loads(b.decode()), Equals(json_data))))
+ elif file_data is not None:
+ matchers.append(AfterPreprocessing(
+ lambda r: multipart.parse_form_data({
+ "REQUEST_METHOD": r.method,
+ "CONTENT_TYPE": r.headers["Content-Type"],
+ "CONTENT_LENGTH": r.headers["Content-Length"],
+ "wsgi.input": io.BytesIO(
+ r.body.read() if hasattr(r.body, "read") else r.body),
+ })[1],
+ MatchesDict(file_data)))
+ if kwargs:
+ matchers.append(MatchesStructure(**kwargs))
+ super().__init__(*matchers)
class TestCharmhubClient(TestCaseWithFactory):
- layer = ZopelessDatabaseLayer
+ layer = LaunchpadZopelessLayer
def setUp(self):
super().setUp()
self.useFixture(FeatureFixture({CHARM_RECIPE_ALLOW_CREATE: "on"}))
- self.pushConfig("charms", charmhub_url="http://charmhub.example/";)
+ self.pushConfig(
+ "charms",
+ charmhub_url="http://charmhub.example/";,
+ charmhub_storage_url="http://storage.charmhub.example/";)
self.client = getUtility(ICharmhubClient)
+ def _setUpSecretStorage(self):
+ self.private_key = PrivateKey.generate()
+ self.pushConfig(
+ "charms",
+ charmhub_secrets_public_key=base64.b64encode(
+ bytes(self.private_key.public_key)).decode(),
+ charmhub_secrets_private_key=base64.b64encode(
+ bytes(self.private_key)).decode())
+
+ def _makeStoreSecrets(self):
+ self.exchanged_key = hashlib.sha256(
+ self.factory.getUniqueBytes()).hexdigest()
+ exchanged_macaroon = Macaroon(key=self.exchanged_key)
+ container = getUtility(IEncryptedContainer, "charmhub-secrets")
+ return {
+ "exchanged_encrypted": removeSecurityProxy(container.encrypt(
+ exchanged_macaroon.serialize().encode())),
+ }
+
+ def _addUnscannedUploadResponse(self):
+ responses.add(
+ "POST", "http://storage.charmhub.example/unscanned-upload/";,
+ json={"successful": True, "upload_id": 1})
+
+ def _addCharmPushResponse(self, name):
+ responses.add(
+ "POST",
+ "http://charmhub.example/v1/charm/{}/revisions".format(
+ quote(name)),
+ status=200,
+ json={
+ "status-url": (
+ "http://charmhub.example/v1/charm/{}/revisions/review";
+ "?upload-id=123".format(quote(name))),
+ })
+
+ def _addCharmReleaseResponse(self, name):
+ responses.add(
+ "POST",
+ "http://charmhub.example/v1/charm/{}/releases".format(quote(name)),
+ json={})
+
@responses.activate
def test_requestPackageUploadPermission(self):
responses.add(
@@ -104,7 +204,7 @@ class TestCharmhubClient(TestCaseWithFactory):
def test_requestPackageUploadPermission_error(self):
responses.add(
"POST", "http://charmhub.example/v1/tokens";,
- status=503, json={"error_list": [{"message": "Failed"}]})
+ status=503, json={"error-list": [{"message": "Failed"}]})
self.assertRaisesWithContent(
BadRequestPackageUploadResponse, "Failed",
self.client.requestPackageUploadPermission, "test-charm")
@@ -161,7 +261,7 @@ class TestCharmhubClient(TestCaseWithFactory):
responses.add(
"POST", "http://charmhub.example/v1/tokens/exchange";,
status=401,
- json={"error_list": [{"message": "Exchange window expired"}]})
+ json={"error-list": [{"message": "Exchange window expired"}]})
root_macaroon_raw = Macaroon(version=2).serialize(JsonSerializer())
discharge_macaroon_raw = Macaroon(version=2).serialize(
JsonSerializer())
@@ -182,3 +282,235 @@ class TestCharmhubClient(TestCaseWithFactory):
"404 Client Error: Not Found",
self.client.exchangeMacaroons,
root_macaroon_raw, discharge_macaroon_raw)
+
+ def makeUploadableCharmRecipeBuild(self, store_secrets=None):
+ if store_secrets is None:
+ store_secrets = self._makeStoreSecrets()
+ recipe = self.factory.makeCharmRecipe(
+ store_upload=True,
+ store_name="test-charm", store_secrets=store_secrets)
+ build = self.factory.makeCharmRecipeBuild(recipe=recipe)
+ charm_lfa = self.factory.makeLibraryFileAlias(
+ filename="test-charm.charm", content="dummy charm content")
+ self.factory.makeCharmFile(build=build, library_file=charm_lfa)
+ manifest_lfa = self.factory.makeLibraryFileAlias(
+ filename="test-charm.manifest", content="dummy manifest content")
+ self.factory.makeCharmFile(build=build, library_file=manifest_lfa)
+ build.updateStatus(BuildStatus.BUILDING)
+ build.updateStatus(BuildStatus.FULLYBUILT)
+ return build
+
+ @responses.activate
+ def test_upload(self):
+ self._setUpSecretStorage()
+ build = self.makeUploadableCharmRecipeBuild()
+ transaction.commit()
+ self._addUnscannedUploadResponse()
+ self._addCharmPushResponse("test-charm")
+ # XXX cjwatson 2021-08-19: Use
+ # config.ICharmhubUploadJobSource.dbuser once that job exists.
+ with dbuser("charm-build-job"):
+ self.assertEqual(
+ "http://charmhub.example/v1/charm/test-charm/revisions/review";
+ "?upload-id=123",
+ self.client.upload(build))
+ requests = [call.request for call in responses.calls]
+ self.assertThat(requests, MatchesListwise([
+ RequestMatches(
+ url=Equals(
+ "http://storage.charmhub.example/unscanned-upload/";),
+ method=Equals("POST"),
+ file_data={
+ "binary": MatchesStructure.byEquality(
+ name="binary", filename="test-charm.charm",
+ value="dummy charm content",
+ content_type="application/octet-stream",
+ )}),
+ RequestMatches(
+ url=Equals(
+ "http://charmhub.example/v1/charm/test-charm/revisions";),
+ method=Equals("POST"),
+ headers=ContainsDict(
+ {"Content-Type": Equals("application/json")}),
+ auth=("Macaroon", MacaroonVerifies(self.exchanged_key)),
+ json_data={"upload-id": 1}),
+ ]))
+
+ @responses.activate
+ def test_upload_unauthorized(self):
+ self._setUpSecretStorage()
+ build = self.makeUploadableCharmRecipeBuild()
+ transaction.commit()
+ self._addUnscannedUploadResponse()
+ charm_push_error = {
+ "code": "permission-required",
+ "message": "Missing required permission: package-manage-revisions",
+ }
+ responses.add(
+ "POST", "http://charmhub.example/v1/charm/test-charm/revisions";,
+ status=401,
+ json={"error-list": [charm_push_error]})
+ # XXX cjwatson 2021-08-19: Use
+ # config.ICharmhubUploadJobSource.dbuser once that job exists.
+ with dbuser("charm-build-job"):
+ self.assertRaisesWithContent(
+ UnauthorizedUploadResponse,
+ "Missing required permission: package-manage-revisions",
+ self.client.upload, build)
+
+ @responses.activate
+ def test_upload_file_error(self):
+ self._setUpSecretStorage()
+ build = self.makeUploadableCharmRecipeBuild()
+ transaction.commit()
+ responses.add(
+ "POST", "http://storage.charmhub.example/unscanned-upload/";,
+ status=502, body="The proxy exploded.\n")
+ # XXX cjwatson 2021-08-19: Use
+ # config.ICharmhubUploadJobSource.dbuser once that job exists.
+ with dbuser("charm-build-job"):
+ err = self.assertRaises(
+ UploadFailedResponse, self.client.upload, build)
+ self.assertEqual("502 Server Error: Bad Gateway", str(err))
+ self.assertThat(err, MatchesStructure(
+ detail=Equals("The proxy exploded.\n"),
+ can_retry=Is(True)))
+
+ @responses.activate
+ def test_checkStatus_pending(self):
+ self._setUpSecretStorage()
+ build = self.makeUploadableCharmRecipeBuild()
+ status_url = (
+ "http://charmhub.example/v1/charm/test-charm/revisions/review";
+ "?upload-id=123")
+ responses.add(
+ "GET", status_url,
+ json={
+ "revisions": [
+ {
+ "upload-id": "123",
+ "status": "new",
+ "revision": None,
+ "errors": None,
+ },
+ ],
+ })
+ self.assertRaises(
+ UploadNotReviewedYetResponse,
+ self.client.checkStatus, build, status_url)
+
+ @responses.activate
+ def test_checkStatus_error(self):
+ self._setUpSecretStorage()
+ build = self.makeUploadableCharmRecipeBuild()
+ status_url = (
+ "http://charmhub.example/v1/charm/test-charm/revisions/review";
+ "?upload-id=123")
+ responses.add(
+ "GET", status_url,
+ json={
+ "revisions": [
+ {
+ "upload-id": "123",
+ "status": "rejected",
+ "revision": None,
+ "errors": [
+ {"code": None, "message": "This charm is broken."},
+ ],
+ },
+ ],
+ })
+ self.assertRaisesWithContent(
+ ReviewFailedResponse, "This charm is broken.",
+ self.client.checkStatus, build, status_url)
+
+ @responses.activate
+ def test_checkStatus_approved(self):
+ self._setUpSecretStorage()
+ build = self.makeUploadableCharmRecipeBuild()
+ status_url = (
+ "http://charmhub.example/v1/charm/test-charm/revisions/review";
+ "?upload-id=123")
+ responses.add(
+ "GET", status_url,
+ json={
+ "revisions": [
+ {
+ "upload-id": "123",
+ "status": "approved",
+ "revision": 1,
+ "errors": None,
+ },
+ ],
+ })
+ self.assertEqual(1, self.client.checkStatus(build, status_url))
+ requests = [call.request for call in responses.calls]
+ self.assertThat(requests, MatchesListwise([
+ RequestMatches(
+ url=Equals(status_url),
+ method=Equals("GET"),
+ auth=("Macaroon", MacaroonVerifies(self.exchanged_key))),
+ ]))
+
+ @responses.activate
+ def test_checkStatus_404(self):
+ self._setUpSecretStorage()
+ build = self.makeUploadableCharmRecipeBuild()
+ status_url = (
+ "http://charmhub.example/v1/charm/test-charm/revisions/review";
+ "?upload-id=123")
+ responses.add("GET", status_url, status=404)
+ self.assertRaisesWithContent(
+ BadReviewStatusResponse, "404 Client Error: Not Found",
+ self.client.checkStatus, build, status_url)
+
+ @responses.activate
+ def test_release(self):
+ self._setUpSecretStorage()
+ recipe = self.factory.makeCharmRecipe(
+ store_upload=True, store_name="test-charm",
+ store_secrets=self._makeStoreSecrets(),
+ store_channels=["stable", "edge"])
+ build = self.factory.makeCharmRecipeBuild(recipe=recipe)
+ self._addCharmReleaseResponse("test-charm")
+ self.client.release(build, 1)
+ self.assertThat(responses.calls[-1].request, RequestMatches(
+ url=Equals("http://charmhub.example/v1/charm/test-charm/releases";),
+ method=Equals("POST"),
+ headers=ContainsDict({"Content-Type": Equals("application/json")}),
+ auth=("Macaroon", MacaroonVerifies(self.exchanged_key)),
+ json_data=[
+ {"channel": "stable", "revision": 1},
+ {"channel": "edge", "revision": 1},
+ ]))
+
+ @responses.activate
+ def test_release_error(self):
+ self._setUpSecretStorage()
+ recipe = self.factory.makeCharmRecipe(
+ store_upload=True, store_name="test-charm",
+ store_secrets=self._makeStoreSecrets(),
+ store_channels=["stable", "edge"])
+ build = self.factory.makeCharmRecipeBuild(recipe=recipe)
+ responses.add(
+ "POST", "http://charmhub.example/v1/charm/test-charm/releases";,
+ status=503,
+ json={"error-list": [{"message": "Failed to publish"}]})
+ self.assertRaisesWithContent(
+ ReleaseFailedResponse, "Failed to publish",
+ self.client.release, build, 1)
+
+ @responses.activate
+ def test_release_404(self):
+ self._setUpSecretStorage()
+ recipe = self.factory.makeCharmRecipe(
+ store_upload=True, store_name="test-charm",
+ store_secrets=self._makeStoreSecrets(),
+ store_channels=["stable", "edge"])
+ build = self.factory.makeCharmRecipeBuild(recipe=recipe)
+ responses.add(
+ "POST", "http://charmhub.example/v1/charm/test-charm/releases";,
+ status=404)
+ self.assertRaisesWithContent(
+ ReleaseFailedResponse, "404 Client Error: Not Found",
+ self.client.release, build, 1)
diff --git a/lib/lp/oci/model/ociregistryclient.py b/lib/lp/oci/model/ociregistryclient.py
index 649b89a..4de58e8 100644
--- a/lib/lp/oci/model/ociregistryclient.py
+++ b/lib/lp/oci/model/ociregistryclient.py
@@ -47,6 +47,7 @@ from lp.oci.interfaces.ociregistryclient import (
)
from lp.services.config import config
from lp.services.features import getFeatureFlag
+from lp.services.librarian.utils import EncodableLibraryFileAlias
from lp.services.propertycache import cachedproperty
from lp.services.timeout import urlfetch
@@ -71,33 +72,6 @@ def is_aws_bearer_token_domain(domain):
return any(domain.endswith(i) for i in domains.split())
-class LibraryFileAliasWrapper:
-
- """A `LibraryFileAlias` wrapper used to read an LFA.
-
- The LFA is uploaded by Buildd to Librarian as tar.gz
- after building the OCI image. Each LFA is essentially
- a docker image layer and it is read in chunks when
- uploading the layer to Dockerhub Registry."""
-
- def __init__(self, lfa):
- self.lfa = lfa
- self.position = 0
-
- def __len__(self):
- return self.lfa.content.filesize - self.position
-
- """ Reads from the LFA in chunks. See ILibraryFileAlias."""
- def read(self, length=-1):
- chunksize = None if length == -1 else length
- data = self.lfa.read(chunksize=chunksize)
- if chunksize is None:
- self.position = self.lfa.content.filesize
- else:
- self.position += length
- return data
-
-
@implementer(IOCIRegistryClient)
class OCIRegistryClient:
@@ -210,7 +184,7 @@ class OCIRegistryClient:
return tarinfo.size
else:
size = lfa.content.filesize
- wrapper = LibraryFileAliasWrapper(lfa)
+ wrapper = EncodableLibraryFileAlias(lfa)
cls._upload(
digest, push_rule, wrapper, size,
http_client)
diff --git a/lib/lp/services/config/schema-lazr.conf b/lib/lp/services/config/schema-lazr.conf
index e6242b9..413e38b 100644
--- a/lib/lp/services/config/schema-lazr.conf
+++ b/lib/lp/services/config/schema-lazr.conf
@@ -155,6 +155,10 @@ cron_control_url: file:cronscripts.ini
# datatype: urlbase
charmhub_url: none
+# Charmhub's storage URL endpoint.
+# datatype: urlbase
+charmhub_storage_url: none
+
# Base64-encoded NaCl private key for decrypting Charmhub upload tokens.
# This should only be set in secret overlays on systems that need to perform
# Charmhub uploads on behalf of users.
diff --git a/lib/lp/services/librarian/utils.py b/lib/lp/services/librarian/utils.py
index 3d627e1..bc0895b 100644
--- a/lib/lp/services/librarian/utils.py
+++ b/lib/lp/services/librarian/utils.py
@@ -1,9 +1,10 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2021 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
__metaclass__ = type
__all__ = [
'copy_and_close',
+ 'EncodableLibraryFileAlias',
'filechunks',
'guess_librarian_encoding',
'sha1_from_path',
@@ -74,3 +75,23 @@ def guess_librarian_encoding(filename, mimetype):
encoding = None
return encoding, mimetype
+
+
+class EncodableLibraryFileAlias:
+ """A `LibraryFileAlias` wrapper usable with a `MultipartEncoder`."""
+
+ def __init__(self, lfa):
+ self.lfa = lfa
+ self.position = 0
+
+ def __len__(self):
+ return self.lfa.content.filesize - self.position
+
+ def read(self, length=-1):
+ chunksize = None if length == -1 else length
+ data = self.lfa.read(chunksize=chunksize)
+ if chunksize is None:
+ self.position = self.lfa.content.filesize
+ else:
+ self.position += length
+ return data
diff --git a/lib/lp/snappy/model/snapstoreclient.py b/lib/lp/snappy/model/snapstoreclient.py
index feba714..fd0415b 100644
--- a/lib/lp/snappy/model/snapstoreclient.py
+++ b/lib/lp/snappy/model/snapstoreclient.py
@@ -1,4 +1,4 @@
-# Copyright 2016-2019 Canonical Ltd. This software is licensed under the
+# Copyright 2016-2021 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Communication with the snap store."""
@@ -29,6 +29,7 @@ from lp.services.crypto.interfaces import (
IEncryptedContainer,
)
from lp.services.features import getFeatureFlag
+from lp.services.librarian.utils import EncodableLibraryFileAlias
from lp.services.memcache.interfaces import IMemcacheClient
from lp.services.scripts import log
from lp.services.timeline.requesttimeline import get_request_timeline
@@ -48,27 +49,6 @@ from lp.snappy.interfaces.snapstoreclient import (
)
-class LibraryFileAliasWrapper:
- """A `LibraryFileAlias` wrapper usable with a `MultipartEncoder`."""
-
- def __init__(self, lfa):
- self.lfa = lfa
- self.position = 0
-
- @property
- def len(self):
- return self.lfa.content.filesize - self.position
-
- def read(self, length=-1):
- chunksize = None if length == -1 else length
- data = self.lfa.read(chunksize=chunksize)
- if chunksize is None:
- self.position = self.lfa.content.filesize
- else:
- self.position += length
- return data
-
-
class InvalidStoreSecretsError(Exception):
pass
@@ -264,7 +244,7 @@ class SnapStoreClient:
config.snappy.store_upload_url, "unscanned-upload/")
lfa.open()
try:
- lfa_wrapper = LibraryFileAliasWrapper(lfa)
+ lfa_wrapper = EncodableLibraryFileAlias(lfa)
encoder = MultipartEncoder(
fields={
"binary": (
diff --git a/setup.py b/setup.py
index 6e945a8..14763f8 100644
--- a/setup.py
+++ b/setup.py
@@ -182,6 +182,7 @@ setup(
'lpjsmin',
'Markdown',
'meliae',
+ 'multipart',
'oauth',
'oauthlib',
'oops',
