launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #20305
[Merge] lp:~cjwatson/launchpad/snap-store-client into lp:launchpad
Colin Watson has proposed merging lp:~cjwatson/launchpad/snap-store-client into lp:launchpad with lp:~cjwatson/launchpad/snap-upload-model as a prerequisite.
Commit message:
Add an initial client for uploading snaps to the store.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
Related bugs:
Bug #1572605 in Launchpad itself: "Automatically upload snap builds to store"
https://bugs.launchpad.net/launchpad/+bug/1572605
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/snap-store-client/+merge/293668
Add an initial client for uploading snaps to the store.
This isn't used anywhere yet, and the implementation will need to change to handle recently-agreed changes to how discharge macaroons are handled, but it's useful to have an outline in place.
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/snap-store-client into lp:launchpad.
=== modified file 'lib/lp/services/config/schema-lazr.conf'
--- lib/lp/services/config/schema-lazr.conf 2016-03-31 14:07:33 +0000
+++ lib/lp/services/config/schema-lazr.conf 2016-05-03 19:57:26 +0000
@@ -1779,6 +1779,12 @@
# datatype: string
tools_source: none
+# The store's primary URL endpoint.
+store_url: none
+
+# The store's upload URL endpoint.
+store_upload_url: none
+
[process-job-source-groups]
# This section is used by cronscripts/process-job-source-groups.py.
dbuser: process-job-source-groups
=== modified file 'lib/lp/snappy/configure.zcml'
--- lib/lp/snappy/configure.zcml 2016-05-03 19:57:25 +0000
+++ lib/lp/snappy/configure.zcml 2016-05-03 19:57:26 +0000
@@ -117,6 +117,13 @@
interface="lp.snappy.interfaces.snapseries.ISnapDistroSeriesSet" />
</securedutility>
+ <!-- Store interaction -->
+ <securedutility
+ class="lp.snappy.model.snapstoreclient.SnapStoreClient"
+ provides="lp.snappy.interfaces.snapstoreclient.ISnapStoreClient">
+ <allow interface="lp.snappy.interfaces.snapstoreclient.ISnapStoreClient" />
+ </securedutility>
+
<webservice:register module="lp.snappy.interfaces.webservice" />
</configure>
=== added file 'lib/lp/snappy/interfaces/snapstoreclient.py'
--- lib/lp/snappy/interfaces/snapstoreclient.py 1970-01-01 00:00:00 +0000
+++ lib/lp/snappy/interfaces/snapstoreclient.py 2016-05-03 19:57:26 +0000
@@ -0,0 +1,43 @@
+# Copyright 2016 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Interface for communication with the snap store."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+ 'BadRequestPackageUploadResponse',
+ 'BadUploadResponse',
+ 'ISnapStoreClient',
+ ]
+
+from zope.interface import Interface
+
+
+class BadRequestPackageUploadResponse(Exception):
+ pass
+
+
+class BadUploadResponse(Exception):
+ pass
+
+
+class ISnapStoreClient(Interface):
+ """Interface for the API provided by the snap store."""
+
+ def requestPackageUpload(snap_series, snap_name):
+ """Request permission from the store to upload builds of a snap.
+
+ :param snap_series: The `ISnapSeries` in which this snap should be
+ published on the store.
+ :param snap_name: The registered name of this snap on the store.
+ :return: A serialized macaroon appropriate for uploading builds of
+ this snap.
+ """
+
+ def upload(snapbuild):
+ """Upload a snap build to the store.
+
+ :param snapbuild: The `ISnapBuild` to upload.
+ """
=== added file 'lib/lp/snappy/model/snapstoreclient.py'
--- lib/lp/snappy/model/snapstoreclient.py 1970-01-01 00:00:00 +0000
+++ lib/lp/snappy/model/snapstoreclient.py 2016-05-03 19:57:26 +0000
@@ -0,0 +1,143 @@
+# Copyright 2016 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Communication with the snap store."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+ 'SnapStoreClient',
+ ]
+
+try:
+ from urllib.parse import quote_plus
+except ImportError:
+ from urllib import quote_plus
+
+from lazr.restful.utils import get_current_browser_request
+import requests
+from requests_toolbelt import MultipartEncoder
+from zope.interface import implementer
+
+from lp.services.config import config
+from lp.services.timeline.requesttimeline import get_request_timeline
+from lp.services.timeout import urlfetch
+from lp.services.webapp.url import urlappend
+from lp.snappy.interfaces.snapstoreclient import (
+ BadRequestPackageUploadResponse,
+ BadUploadResponse,
+ ISnapStoreClient,
+ )
+
+
+class LibraryFileAliasWrapper:
+ """A `LibraryFileAlias` wrapper usable with a `MultipartEncoder`."""
+
+ def __init__(self, lfa):
+ self.lfa = lfa
+ self.position = 0
+
+ @property
+ def len(self):
+ return self.lfa.content.filesize - self.position
+
+ def read(self, length=-1):
+ chunksize = None if length == -1 else length
+ data = self.lfa.read(chunksize=chunksize)
+ if chunksize is None:
+ self.position = self.lfa.content.filesize
+ else:
+ self.position += length
+ return data
+
+
+class MacaroonAuth(requests.auth.AuthBase):
+ """Attaches macaroon authentication to a given Request object."""
+
+ def __init__(self, tokens):
+ self.tokens = tokens
+
+ def __call__(self, r):
+ r.headers["Authorization"] = (
+ "Macaroon " +
+ ", ".join('%s="%s"' % (k, v) for k, v in self.tokens.items()))
+ return r
+
+
+@implementer(ISnapStoreClient)
+class SnapStoreClient:
+ """A client for the API provided by the snap store."""
+
+ def requestPackageUpload(self, snap_series, snap_name):
+ assert config.snappy.store_url is not None
+ request_url = urlappend(
+ config.snappy.store_url, "api/2.0/acl/package_upload/")
+ request = get_current_browser_request()
+ timeline_action = get_request_timeline(request).start(
+ "request-snap-upload-macaroon",
+ "%s/%s" % (snap_series.name, snap_name), allow_nested=True)
+ try:
+ response = urlfetch(
+ request_url, method="POST",
+ json={"name": snap_name, "series": snap_series.name})
+ response_data = response.json()
+ if "macaroon" not in response_data:
+ raise BadRequestPackageUploadResponse(response.text)
+ return response_data["macaroon"]
+ except requests.HTTPError as e:
+ raise BadRequestPackageUploadResponse(e.response.text)
+ finally:
+ timeline_action.finish()
+
+ def _uploadFile(self, lfa, lfc):
+ """Upload a single file."""
+ assert config.snappy.store_upload_url is not None
+ unscanned_upload_url = urlappend(
+ config.snappy.store_upload_url, "unscanned-upload/")
+ lfa.open()
+ try:
+ lfa_wrapper = LibraryFileAliasWrapper(lfa)
+ encoder = MultipartEncoder(
+ fields={
+ "binary": (
+ "filename", lfa_wrapper, "application/octet-stream"),
+ })
+ try:
+ response = urlfetch(
+ unscanned_upload_url, method="POST", data=encoder,
+ headers={"Content-Type": encoder.content_type})
+ response_data = response.json()
+ if not response_data.get("successful", False):
+ raise BadUploadResponse(response.text)
+ return {"upload_id": response_data["upload_id"]}
+ except requests.HTTPError as e:
+ raise BadUploadResponse(e.response.text)
+ finally:
+ lfa.close()
+
+ def _uploadApp(self, snap, upload_data):
+ """Create a new store upload based on the uploaded file."""
+ assert config.snappy.store_url is not None
+ assert snap.store_name is not None
+ upload_url = urlappend(
+ config.snappy.store_url,
+ "dev/api/snap-upload/%s/" % quote_plus(snap.store_name))
+ data = {
+ "updown_id": upload_data["upload_id"],
+ "series": snap.store_series.name,
+ }
+ # XXX cjwatson 2016-04-20: handle refresh
+ try:
+ assert snap.store_tokens is not None
+ urlfetch(
+ upload_url, method="POST", data=data,
+ auth=MacaroonAuth(snap.store_tokens))
+ except requests.HTTPError as e:
+ raise BadUploadResponse(e.response.text)
+
+ def upload(self, snapbuild):
+ """See `ISnapStoreClient`."""
+ for _, lfa, lfc in snapbuild.getFiles():
+ upload_data = self._uploadFile(lfa, lfc)
+ self._uploadApp(snapbuild.snap, upload_data)
=== added file 'lib/lp/snappy/tests/test_snapstoreclient.py'
--- lib/lp/snappy/tests/test_snapstoreclient.py 1970-01-01 00:00:00 +0000
+++ lib/lp/snappy/tests/test_snapstoreclient.py 2016-05-03 19:57:26 +0000
@@ -0,0 +1,183 @@
+# Copyright 2016 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Tests for communication with the snap store."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+
+from cgi import FieldStorage
+import io
+import json
+
+from httmock import (
+ all_requests,
+ HTTMock,
+ urlmatch,
+ )
+from requests.utils import parse_dict_header
+from testtools.matchers import (
+ Contains,
+ Equals,
+ Matcher,
+ MatchesDict,
+ MatchesStructure,
+ StartsWith,
+ )
+import transaction
+from zope.component import getUtility
+
+from lp.services.config import config
+from lp.services.features.testing import FeatureFixture
+from lp.services.webapp.url import urlappend
+from lp.snappy.interfaces.snap import SNAP_TESTING_FLAGS
+from lp.snappy.interfaces.snapstoreclient import (
+ BadRequestPackageUploadResponse,
+ ISnapStoreClient,
+ )
+from lp.testing import TestCaseWithFactory
+from lp.testing.layers import LaunchpadZopelessLayer
+
+
+class RequestMatches(Matcher):
+ """Matches a request with the specified attributes."""
+
+ def __init__(self, url, auth=None, json_data=None, form_data=None,
+ **kwargs):
+ self.url = url
+ self.auth = auth
+ self.json_data = json_data
+ self.form_data = form_data
+ self.kwargs = kwargs
+
+ def match(self, request):
+ mismatch = MatchesStructure(url=self.url, **self.kwargs).match(request)
+ if mismatch is not None:
+ return mismatch
+ if self.auth is not None:
+ mismatch = Contains("Authorization").match(request.headers)
+ if mismatch is not None:
+ return mismatch
+ auth_value = request.headers["Authorization"]
+ auth_scheme, auth_params = self.auth
+ mismatch = StartsWith(auth_scheme + " ").match(auth_value)
+ if mismatch is not None:
+ return mismatch
+ mismatch = Equals(auth_params).match(
+ parse_dict_header(auth_value[len(auth_scheme + " "):]))
+ if mismatch is not None:
+ return mismatch
+ if self.json_data is not None:
+ mismatch = Equals(self.json_data).match(json.loads(request.body))
+ if mismatch is not None:
+ return mismatch
+ if self.form_data is not None:
+ if hasattr(request.body, "read"):
+ body = request.body.read()
+ else:
+ body = request.body
+ fs = FieldStorage(
+ fp=io.BytesIO(body),
+ environ={"REQUEST_METHOD": request.method},
+ headers=request.headers)
+ mismatch = MatchesDict(self.form_data).match(fs)
+ if mismatch is not None:
+ return mismatch
+
+
+class TestSnapStoreClient(TestCaseWithFactory):
+
+ layer = LaunchpadZopelessLayer
+
+ def setUp(self):
+ super(TestSnapStoreClient, self).setUp()
+ self.useFixture(FeatureFixture(SNAP_TESTING_FLAGS))
+ self.pushConfig(
+ "snappy", store_url="http://sca.example/";,
+ store_upload_url="http://updown.example/";)
+ self.client = getUtility(ISnapStoreClient)
+
+ def test_requestPackageUpload(self):
+ @all_requests
+ def handler(url, request):
+ self.request = request
+ return {"status_code": 200, "content": {"macaroon": "dummy"}}
+
+ snap_series = self.factory.makeSnapSeries(name="rolling")
+ with HTTMock(handler):
+ macaroon = self.client.requestPackageUpload(
+ snap_series, "test-snap")
+ self.assertThat(self.request, RequestMatches(
+ url=Equals(urlappend(
+ config.snappy.store_url, "api/2.0/acl/package_upload/")),
+ method=Equals("POST"),
+ json_data={"name": "test-snap", "series": "rolling"}))
+ self.assertEqual("dummy", macaroon)
+
+ def test_requestPackageUpload_missing_macaroon(self):
+ @all_requests
+ def handler(url, request):
+ return {"status_code": 200, "content": {}}
+
+ snap_series = self.factory.makeSnapSeries()
+ with HTTMock(handler):
+ self.assertRaises(
+ BadRequestPackageUploadResponse,
+ self.client.requestPackageUpload, snap_series, "test-snap")
+
+ def test_requestPackageUpload_404(self):
+ @all_requests
+ def handler(url, request):
+ return {"status_code": 404}
+
+ snap_series = self.factory.makeSnapSeries()
+ with HTTMock(handler):
+ self.assertRaises(
+ BadRequestPackageUploadResponse,
+ self.client.requestPackageUpload, snap_series, "test-snap")
+
+ def test_upload(self):
+ @urlmatch(path=r".*/unscanned-upload/$")
+ def unscanned_upload_handler(url, request):
+ self.unscanned_upload_request = request
+ return {
+ "status_code": 200,
+ "content": {"successful": True, "upload_id": 1},
+ }
+
+ @urlmatch(path=r".*/snap-upload/.*")
+ def snap_upload_handler(url, request):
+ self.snap_upload_request = request
+ return {"status_code": 202, "content": {"success": True}}
+
+ store_tokens = {"root": "dummy-root", "discharge": "dummy-discharge"}
+ snap = self.factory.makeSnap(
+ store_upload=True,
+ store_series=self.factory.makeSnapSeries(name="rolling"),
+ store_name="test-snap", store_tokens=store_tokens)
+ snapbuild = self.factory.makeSnapBuild(snap=snap)
+ lfa = self.factory.makeLibraryFileAlias(content="dummy snap content")
+ self.factory.makeSnapFile(snapbuild=snapbuild, libraryfile=lfa)
+ transaction.commit()
+ with HTTMock(unscanned_upload_handler, snap_upload_handler):
+ self.client.upload(snapbuild)
+ self.assertThat(self.unscanned_upload_request, RequestMatches(
+ url=Equals(urlappend(
+ config.snappy.store_upload_url, "unscanned-upload/")),
+ method=Equals("POST"),
+ form_data={
+ "binary": MatchesStructure.byEquality(
+ name="binary", filename="filename",
+ value="dummy snap content",
+ type="application/octet-stream",
+ )}))
+ self.assertThat(self.snap_upload_request, RequestMatches(
+ url=Equals(urlappend(
+ config.snappy.store_url, "dev/api/snap-upload/test-snap/")),
+ method=Equals("POST"), auth=("Macaroon", store_tokens),
+ form_data={
+ "updown_id": MatchesStructure.byEquality(
+ name="updown_id", value="1"),
+ "series": MatchesStructure.byEquality(
+ name="series", value="rolling")}))
=== modified file 'setup.py'
--- setup.py 2016-03-16 02:08:40 +0000
+++ setup.py 2016-05-03 19:57:26 +0000
@@ -77,13 +77,14 @@
'paramiko',
'pgbouncer',
'psycopg2',
- 'python-memcached',
'pyasn1',
'pystache',
+ 'python-memcached',
'python-openid',
'pytz',
'rabbitfixture',
'requests',
+ 'requests-toolbelt',
's4',
'setproctitle',
'setuptools',
=== modified file 'versions.cfg'
--- versions.cfg 2016-03-16 02:08:40 +0000
+++ versions.cfg 2016-05-03 19:57:26 +0000
@@ -119,6 +119,7 @@
PyYAML = 3.10
rabbitfixture = 0.3.6
requests = 2.7.0
+requests-toolbelt = 0.6.0
s4 = 0.1.2
setproctitle = 1.1.7
setuptools-git = 1.0
Follow ups
