launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #20698
[Merge] lp:~cjwatson/launchpad/snap-channels-store-client into lp:launchpad
Colin Watson has proposed merging lp:~cjwatson/launchpad/snap-channels-store-client into lp:launchpad with lp:~cjwatson/launchpad/snap-auto-build-ui as a prerequisite.
Commit message:
Add model and store client support for automatically releasing snap packages.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
Related bugs:
Bug #1597819 in Launchpad itself: "Add option to automatically release snap packages to channels after upload"
https://bugs.launchpad.net/launchpad/+bug/1597819
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/snap-channels-store-client/+merge/298806
Add model and store client support for automatically releasing snap packages.
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/snap-channels-store-client into lp:launchpad.
=== modified file 'configs/development/launchpad-lazr.conf'
--- configs/development/launchpad-lazr.conf 2016-05-18 00:33:18 +0000
+++ configs/development/launchpad-lazr.conf 2016-06-30 16:26:55 +0000
@@ -190,6 +190,7 @@
builder_proxy_auth_api_endpoint: http://snap-proxy.launchpad.dev:8080/tokens
builder_proxy_host: snap-proxy.launchpad.dev
builder_proxy_port: 3128
+store_search_url: https://search.apps.ubuntu.com/
tools_source: deb http://ppa.launchpad.net/snappy-dev/snapcraft-daily/ubuntu %(series)s main
[txlongpoll]
=== modified file 'lib/lp/services/config/schema-lazr.conf'
--- lib/lp/services/config/schema-lazr.conf 2016-06-20 20:18:11 +0000
+++ lib/lp/services/config/schema-lazr.conf 2016-06-30 16:26:55 +0000
@@ -1804,6 +1804,9 @@
# The store's upload URL endpoint.
store_upload_url: none
+# The store's search URL endpoint.
+store_search_url: none
+
[process-job-source-groups]
# This section is used by cronscripts/process-job-source-groups.py.
dbuser: process-job-source-groups
=== modified file 'lib/lp/snappy/interfaces/snap.py'
--- lib/lp/snappy/interfaces/snap.py 2016-06-20 20:32:36 +0000
+++ lib/lp/snappy/interfaces/snap.py 2016-06-30 16:26:55 +0000
@@ -435,6 +435,13 @@
"Serialized secrets issued by the store and the login service to "
"authorize uploads of this snap package."))
+ store_channels = List(
+ value_type=TextLine(), title=_("Store channels"),
+ required=False, readonly=False,
+ description=_(
+ "Channels to which to release this snap package after "
+ "uploading it to the store."))
+
class ISnapAdminAttributes(Interface):
"""`ISnap` attributes that can be edited by admins.
=== modified file 'lib/lp/snappy/interfaces/snapstoreclient.py'
--- lib/lp/snappy/interfaces/snapstoreclient.py 2016-06-21 14:51:06 +0000
+++ lib/lp/snappy/interfaces/snapstoreclient.py 2016-06-30 16:26:55 +0000
@@ -8,11 +8,14 @@
__metaclass__ = type
__all__ = [
'BadRefreshResponse',
+ 'BadReleaseResponse',
'BadRequestPackageUploadResponse',
'BadScanStatusResponse',
+ 'BadSearchResponse',
'BadUploadResponse',
'ISnapStoreClient',
'NeedsRefreshResponse',
+ 'ReleaseFailedResponse',
'ScanFailedResponse',
'UnauthorizedUploadResponse',
'UploadNotScannedYetResponse',
@@ -53,6 +56,18 @@
pass
+class BadSearchResponse(Exception):
+ pass
+
+
+class BadReleaseResponse(Exception):
+ pass
+
+
+class ReleaseFailedResponse(Exception):
+ pass
+
+
class ISnapStoreClient(Interface):
"""Interface for the API provided by the snap store."""
@@ -91,6 +106,26 @@
scanned the upload.
:raises BadScanStatusResponse: if the store failed to scan the
upload.
- :return: A URL on the store with further information about this
- upload.
+ :return: A tuple of (`url`, `revision`), where `url` is a URL on the
+ store with further information about this upload, and `revision`
+ is the store revision number for the upload or None.
+ """
+
+ def listChannels():
+ """Fetch the current list of channels from the store.
+
+ :raises BadSearchResponse: if the attempt to fetch the list of
+ channels from the store fails.
+ :return: A list of dictionaries, one per channel, each of which
+ contains at least "name" and "display_name" keys.
+ """
+
+ def release(snapbuild, revision):
+ """Tell the store to release a snap build to specified channels.
+
+ :param snapbuild: The `ISnapBuild` to release.
+ :param revision: The revision returned by the store when uploading
+ the build.
+ :raises BadReleaseResponse: if the store failed to release the
+ build.
"""
=== modified file 'lib/lp/snappy/model/snap.py'
--- lib/lp/snappy/model/snap.py 2016-06-30 16:09:25 +0000
+++ lib/lp/snappy/model/snap.py 2016-06-30 16:26:55 +0000
@@ -187,12 +187,14 @@
store_secrets = JSON('store_secrets', allow_none=True)
+ store_channels = JSON('store_channels', allow_none=True)
+
def __init__(self, registrant, owner, distro_series, name,
description=None, branch=None, git_ref=None, auto_build=False,
auto_build_archive=None, auto_build_pocket=None,
require_virtualized=True, date_created=DEFAULT,
private=False, store_upload=False, store_series=None,
- store_name=None, store_secrets=None):
+ store_name=None, store_secrets=None, store_channels=None):
"""Construct a `Snap`."""
if not getFeatureFlag(SNAP_FEATURE_FLAG):
raise SnapFeatureDisabled
@@ -216,6 +218,7 @@
self.store_series = store_series
self.store_name = store_name
self.store_secrets = store_secrets
+ self.store_channels = store_channels
@property
def valid_webhook_event_types(self):
@@ -511,7 +514,7 @@
auto_build_archive=None, auto_build_pocket=None,
require_virtualized=True, processors=None, date_created=DEFAULT,
private=False, store_upload=False, store_series=None,
- store_name=None, store_secrets=None):
+ store_name=None, store_secrets=None, store_channels=None):
"""See `ISnapSet`."""
if not registrant.inTeam(owner):
if owner.is_team:
@@ -540,7 +543,7 @@
require_virtualized=require_virtualized, date_created=date_created,
private=private, store_upload=store_upload,
store_series=store_series, store_name=store_name,
- store_secrets=store_secrets)
+ store_secrets=store_secrets, store_channels=store_channels)
store.add(snap)
if processors is None:
=== modified file 'lib/lp/snappy/model/snapstoreclient.py'
--- lib/lp/snappy/model/snapstoreclient.py 2016-06-27 13:19:15 +0000
+++ lib/lp/snappy/model/snapstoreclient.py 2016-06-30 16:26:55 +0000
@@ -10,25 +10,36 @@
'SnapStoreClient',
]
+import json
import string
+import time
+from urlparse import urlsplit
from lazr.restful.utils import get_current_browser_request
from pymacaroons import Macaroon
import requests
from requests_toolbelt import MultipartEncoder
+from zope.component import getUtility
from zope.interface import implementer
+from zope.security.proxy import removeSecurityProxy
from lp.services.config import config
+from lp.services.features import getFeatureFlag
+from lp.services.memcache.interfaces import IMemcacheClient
+from lp.services.scripts import log
from lp.services.timeline.requesttimeline import get_request_timeline
from lp.services.timeout import urlfetch
from lp.services.webapp.url import urlappend
from lp.snappy.interfaces.snapstoreclient import (
BadRefreshResponse,
+ BadReleaseResponse,
BadRequestPackageUploadResponse,
BadScanStatusResponse,
+ BadSearchResponse,
BadUploadResponse,
ISnapStoreClient,
NeedsRefreshResponse,
+ ReleaseFailedResponse,
ScanFailedResponse,
UnauthorizedUploadResponse,
UploadNotScannedYetResponse,
@@ -99,10 +110,28 @@
return r
+# Hardcoded fallback.
+_default_store_channels = [
+ {"name": "candidate", "display_name": "Candidate"},
+ {"name": "edge", "display_name": "Edge"},
+ {"name": "beta", "display_name": "Beta"},
+ {"name": "stable", "display_name": "Stable"},
+ ]
+
+
@implementer(ISnapStoreClient)
class SnapStoreClient:
"""A client for the API provided by the snap store."""
+ @staticmethod
+ def _getTimeline():
+ # XXX cjwatson 2016-06-29: This can be simplified once jobs have
+ # timeline support.
+ request = get_current_browser_request()
+ if request is None:
+ return None
+ return get_request_timeline(request)
+
def requestPackageUploadPermission(self, snappy_series, snap_name):
assert config.snappy.store_url is not None
request_url = urlappend(config.snappy.store_url, "dev/api/acl/")
@@ -203,6 +232,7 @@
@classmethod
def refreshDischargeMacaroon(cls, snap):
+ """See `ISnapStoreClient`."""
assert config.launchpad.openid_provider_root is not None
assert snap.store_secrets is not None
refresh_url = urlappend(
@@ -222,6 +252,7 @@
@classmethod
def checkStatus(cls, status_url):
+ """See `ISnapStoreClient`."""
try:
response = urlfetch(status_url)
response_data = response.json()
@@ -232,7 +263,9 @@
elif not response_data["application_url"]:
raise ScanFailedResponse(response_data["message"])
else:
- return response_data["application_url"]
+ return (
+ response_data["application_url"],
+ response_data["revision"])
else:
# New status format.
if not response_data["processed"]:
@@ -241,7 +274,88 @@
error_message = "\n".join(
error["message"] for error in response_data["errors"])
raise ScanFailedResponse(error_message)
+ elif not response_data["can_release"]:
+ return response_data["url"], None
else:
- return response_data["url"]
+ return response_data["url"], response_data["revision"]
except requests.HTTPError as e:
raise BadScanStatusResponse(e.args[0])
+
+ @classmethod
+ def listChannels(cls):
+ """See `ISnapStoreClient`."""
+ if config.snappy.store_search_url is None:
+ return _default_store_channels
+ channels = None
+ memcache_client = getUtility(IMemcacheClient)
+ search_host = urlsplit(config.snappy.store_search_url).hostname
+ memcache_key = ("%s:channels" % search_host).encode("UTF-8")
+ cached_channels = memcache_client.get(memcache_key)
+ if cached_channels is not None:
+ try:
+ channels = json.loads(cached_channels)
+ except Exception:
+ log.exception(
+ "Cannot load cached channels for %s; deleting" %
+ search_host)
+ memcache_client.delete(memcache_key)
+ if (channels is None and
+ not getFeatureFlag(u"snap.disable_channel_search")):
+ path = "api/v1/channels"
+ timeline = cls._getTimeline()
+ if timeline is not None:
+ action = timeline.start("store-search-get", "/" + path)
+ channels_url = urlappend(config.snappy.store_search_url, path)
+ try:
+ response = urlfetch(
+ channels_url, headers={"Accept": "application/hal+json"})
+ except requests.HTTPError as e:
+ raise BadSearchResponse(e.args[0])
+ finally:
+ if timeline is not None:
+ action.finish()
+ channels = response.json().get("_embedded", {}).get(
+ "clickindex:channel", [])
+ expire_time = time.time() + 60 * 60 * 24
+ memcache_client.set(
+ memcache_key, json.dumps(channels), expire_time)
+ if channels is None:
+ channels = _default_store_channels
+ return channels
+
+ @classmethod
+ def release(cls, snapbuild, revision):
+ """See `ISnapStoreClient`."""
+ assert config.snappy.store_url is not None
+ snap = snapbuild.snap
+ assert snap.store_name is not None
+ assert snap.store_series is not None
+ assert snap.store_channels
+ release_url = urlappend(
+ config.snappy.store_url, "dev/api/snap-release/")
+ data = {
+ "name": snap.store_name,
+ "revision": revision,
+ # The security proxy is useless and breaks JSON serialisation.
+ "channels": removeSecurityProxy(snap.store_channels),
+ "series": snap.store_series.name,
+ }
+ # XXX cjwatson 2016-06-28: This should add timeline information, but
+ # that's currently difficult in jobs.
+ try:
+ assert snap.store_secrets is not None
+ urlfetch(
+ release_url, method="POST", json=data,
+ auth=MacaroonAuth(
+ snap.store_secrets["root"],
+ snap.store_secrets["discharge"]))
+ except requests.HTTPError as e:
+ if e.response is not None:
+ error = None
+ try:
+ error = e.response.json()["errors"]
+ except Exception:
+ pass
+ if error is not None:
+ raise ReleaseFailedResponse(error)
+ raise BadReleaseResponse(e.args[0])
=== modified file 'lib/lp/snappy/tests/test_snapstoreclient.py'
--- lib/lp/snappy/tests/test_snapstoreclient.py 2016-06-27 13:19:15 +0000
+++ lib/lp/snappy/tests/test_snapstoreclient.py 2016-06-30 16:26:55 +0000
@@ -39,12 +39,16 @@
from zope.component import getUtility
from lp.services.features.testing import FeatureFixture
+from lp.services.memcache.interfaces import IMemcacheClient
from lp.services.timeline.requesttimeline import get_request_timeline
from lp.snappy.interfaces.snap import SNAP_TESTING_FLAGS
from lp.snappy.interfaces.snapstoreclient import (
+ BadReleaseResponse,
BadRequestPackageUploadResponse,
BadScanStatusResponse,
+ BadSearchResponse,
ISnapStoreClient,
+ ReleaseFailedResponse,
ScanFailedResponse,
UnauthorizedUploadResponse,
UploadNotScannedYetResponse,
@@ -169,7 +173,8 @@
self.useFixture(FeatureFixture(SNAP_TESTING_FLAGS))
self.pushConfig(
"snappy", store_url="http://sca.example/";,
- store_upload_url="http://updown.example/";)
+ store_upload_url="http://updown.example/";,
+ store_search_url="http://search.example/";)
self.pushConfig(
"launchpad", openid_provider_root="http://sso.example/";)
self.client = getUtility(ISnapStoreClient)
@@ -225,6 +230,22 @@
"content": {"discharge_macaroon": new_macaroon.serialize()},
}
+ @urlmatch(path=r".*/snap-release/$")
+ def _snap_release_handler(self, url, request):
+ self.snap_release_request = request
+ return {
+ "status_code": 200,
+ "content": {
+ "success": True,
+ "channel_map": [
+ {"channel": "stable", "info": "specific",
+ "version": "1.0", "revision": 1},
+ {"channel": "edge", "info": "specific",
+ "version": "1.0", "revision": 1},
+ ],
+ "opened_channels": ["stable", "edge"],
+ }}
+
def test_requestPackageUploadPermission(self):
@all_requests
def handler(url, request):
@@ -452,7 +473,7 @@
status_url = "http://sca.example/dev/api/click-scan-complete/updown/1/";
with HTTMock(handler):
self.assertEqual(
- "http://sca.example/dev/click-apps/1/";,
+ ("http://sca.example/dev/click-apps/1/";, 1),
self.client.checkStatus(status_url))
def test_checkStatus_new_pending(self):
@@ -525,7 +546,7 @@
status_url = "http://sca.example/dev/api/snaps/1/builds/1/status";
with HTTMock(handler):
self.assertEqual(
- "http://sca.example/dev/click-apps/1/rev/1/";,
+ ("http://sca.example/dev/click-apps/1/rev/1/";, 1),
self.client.checkStatus(status_url))
def test_checkStatus_404(self):
@@ -538,3 +559,121 @@
self.assertRaisesWithContent(
BadScanStatusResponse, b"404 Client Error: Not found",
self.client.checkStatus, status_url)
+
+ def test_listChannels(self):
+ expected_channels = [
+ {"name": "stable", "display_name": "Stable"},
+ {"name": "edge", "display_name": "Edge"},
+ ]
+
+ @all_requests
+ def handler(url, request):
+ self.request = request
+ return {
+ "status_code": 200,
+ "content": {
+ "_embedded": {"clickindex:channel": expected_channels}}}
+
+ memcache_key = "search.example:channels".encode("UTF-8")
+ try:
+ with HTTMock(handler):
+ self.assertEqual(expected_channels, self.client.listChannels())
+ self.assertThat(self.request, RequestMatches(
+ url=Equals("http://search.example/api/v1/channels";),
+ method=Equals("GET"),
+ headers=ContainsDict(
+ {"Accept": Equals("application/hal+json")})))
+ self.assertEqual(
+ expected_channels,
+ json.loads(getUtility(IMemcacheClient).get(memcache_key)))
+ self.request = None
+ with HTTMock(handler):
+ self.assertEqual(expected_channels, self.client.listChannels())
+ self.assertIsNone(self.request)
+ finally:
+ getUtility(IMemcacheClient).delete(memcache_key)
+
+ def test_listChannels_404(self):
+ @all_requests
+ def handler(url, request):
+ return {"status_code": 404, "reason": b"Not found"}
+
+ with HTTMock(handler):
+ self.assertRaisesWithContent(
+ BadSearchResponse, b"404 Client Error: Not found",
+ self.client.listChannels)
+
+ def test_listChannels_disable_search(self):
+ @all_requests
+ def handler(url, request):
+ self.request = request
+ return {"status_code": 404, "reason": b"Not found"}
+
+ self.useFixture(
+ FeatureFixture({u"snap.disable_channel_search": u"on"}))
+ expected_channels = [
+ {"name": "candidate", "display_name": "Candidate"},
+ {"name": "edge", "display_name": "Edge"},
+ {"name": "beta", "display_name": "Beta"},
+ {"name": "stable", "display_name": "Stable"},
+ ]
+ self.request = None
+ with HTTMock(handler):
+ self.assertEqual(expected_channels, self.client.listChannels())
+ self.assertIsNone(self.request)
+ memcache_key = "search.example:channels".encode("UTF-8")
+ self.assertIsNone(getUtility(IMemcacheClient).get(memcache_key))
+
+ def test_release(self):
+ snap = self.factory.makeSnap(
+ store_upload=True,
+ store_series=self.factory.makeSnappySeries(name="rolling"),
+ store_name="test-snap", store_secrets=self._make_store_secrets(),
+ store_channels=["stable", "edge"])
+ snapbuild = self.factory.makeSnapBuild(snap=snap)
+ with HTTMock(self._snap_release_handler):
+ self.client.release(snapbuild, 1)
+ self.assertThat(self.snap_release_request, RequestMatches(
+ url=Equals("http://sca.example/dev/api/snap-release/";),
+ method=Equals("POST"),
+ headers=ContainsDict({"Content-Type": Equals("application/json")}),
+ auth=("Macaroon", MacaroonsVerify(self.root_key)),
+ json_data={
+ "name": "test-snap", "revision": 1,
+ "channels": ["stable", "edge"], "series": "rolling",
+ }))
+
+ def test_release_error(self):
+ @urlmatch(path=r".*/snap-release/$")
+ def handler(url, request):
+ return {
+ "status_code": 503,
+ "content": {"success": False, "errors": "Failed to publish"},
+ }
+
+ snap = self.factory.makeSnap(
+ store_upload=True,
+ store_series=self.factory.makeSnappySeries(name="rolling"),
+ store_name="test-snap", store_secrets=self._make_store_secrets(),
+ store_channels=["stable", "edge"])
+ snapbuild = self.factory.makeSnapBuild(snap=snap)
+ with HTTMock(handler):
+ self.assertRaisesWithContent(
+ ReleaseFailedResponse, "Failed to publish",
+ self.client.release, snapbuild, 1)
+
+ def test_release_404(self):
+ @urlmatch(path=r".*/snap-release/$")
+ def handler(url, request):
+ return {"status_code": 404, "reason": b"Not found"}
+
+ snap = self.factory.makeSnap(
+ store_upload=True,
+ store_series=self.factory.makeSnappySeries(name="rolling"),
+ store_name="test-snap", store_secrets=self._make_store_secrets(),
+ store_channels=["stable", "edge"])
+ snapbuild = self.factory.makeSnapBuild(snap=snap)
+ with HTTMock(handler):
+ self.assertRaisesWithContent(
+ BadReleaseResponse, b"404 Client Error: Not found",
+ self.client.release, snapbuild, 1)
=== modified file 'lib/lp/testing/factory.py'
--- lib/lp/testing/factory.py 2016-06-20 20:18:11 +0000
+++ lib/lp/testing/factory.py 2016-06-30 16:26:55 +0000
@@ -4634,7 +4634,8 @@
auto_build_archive=None, auto_build_pocket=None,
is_stale=None, require_virtualized=True, processors=None,
date_created=DEFAULT, private=False, store_upload=False,
- store_series=None, store_name=None, store_secrets=None):
+ store_series=None, store_name=None, store_secrets=None,
+ store_channels=None):
"""Make a new Snap."""
if registrant is None:
registrant = self.makePerson()
@@ -4659,7 +4660,8 @@
auto_build=auto_build, auto_build_archive=auto_build_archive,
auto_build_pocket=auto_build_pocket, private=private,
store_upload=store_upload, store_series=store_series,
- store_name=store_name, store_secrets=store_secrets)
+ store_name=store_name, store_secrets=store_secrets,
+ store_channels=store_channels)
if is_stale is not None:
removeSecurityProxy(snap).is_stale = is_stale
IStore(snap).flush()
Follow ups
