launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #30951
[Merge] ~pelpsi/launchpad:fetch-service-session-API into launchpad:master
Simone Pelosi has proposed merging ~pelpsi/launchpad:fetch-service-session-API into launchpad:master with ~ines-almeida/launchpad:fetch-service-option-model-update as a prerequisite.
Commit message:
Add fetch service session API
Update BuilderProxyMixin class to switch between
Fetch Service and classic Builder Proxy depending on
use_fetch_service flag.
Add test suite for Fetch Service and fetchservice.py mock class
for API.
Add Fetch Service API information on schema-lazr.conf.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~pelpsi/launchpad/+git/launchpad/+merge/461721
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~pelpsi/launchpad:fetch-service-session-API into launchpad:master.
diff --git a/lib/lp/buildmaster/builderproxy.py b/lib/lp/buildmaster/builderproxy.py
index be65e29..4f249a7 100644
--- a/lib/lp/buildmaster/builderproxy.py
+++ b/lib/lp/buildmaster/builderproxy.py
@@ -20,7 +20,10 @@ from typing import Dict, Generator
from twisted.internet import defer
-from lp.buildmaster.downloader import RequestProxyTokenCommand
+from lp.buildmaster.downloader import (
+ RequestFetchServiceSessionCommand,
+ RequestProxyTokenCommand,
+)
from lp.buildmaster.interfaces.builder import CannotBuild
from lp.buildmaster.interfaces.buildfarmjobbehaviour import BuildArgs
from lp.services.config import config
@@ -36,9 +39,16 @@ class BuilderProxyMixin:
@defer.inlineCallbacks
def addProxyArgs(
- self, args: BuildArgs, allow_internet: bool = True
+ self,
+ args: BuildArgs,
+ allow_internet: bool = True,
+ fetch_service: bool = False,
) -> Generator[None, Dict[str, str], None]:
- if _get_proxy_config("builder_proxy_host") and allow_internet:
+ if (
+ not fetch_service
+ and _get_proxy_config("builder_proxy_host")
+ and allow_internet
+ ):
token = yield self._requestProxyToken()
args["proxy_url"] = (
"http://{username}:{password}@{host}:{port}".format(
@@ -52,6 +62,25 @@ class BuilderProxyMixin:
endpoint=_get_proxy_config("builder_proxy_auth_api_endpoint"),
token=token["username"],
)
+ if (
+ fetch_service
+ and _get_proxy_config("fetch_service_host")
+ and allow_internet
+ ):
+ # http://<session-id>:<secret>@<addr>:9988/
+ token = yield self._requestFetchServiceSession()
+ args["proxy_url"] = (
+ "http://{session_id}:{token}@{host}:{port}".format(
+ session_id=token["id"],
+ token=token["token"],
+ host=_get_proxy_config("fetch_service_host"),
+ port=_get_proxy_config("fetch_service_port"),
+ )
+ )
+ args["revocation_endpoint"] = "{endpoint}/session/{id}".format(
+ endpoint=_get_proxy_config("fetch_service_auth_api_endpoint"),
+ id=token["id"],
+ )
@defer.inlineCallbacks
def _requestProxyToken(self):
@@ -86,3 +115,39 @@ class BuilderProxyMixin:
proxy_username=proxy_username,
)
return token
+
+ @defer.inlineCallbacks
+ def _requestFetchServiceSession(self):
+ # This is a different function if we have the needs to
+ # differentiate more the behavior.
+ admin_username = _get_proxy_config(
+ "fetch_service_auth_api_admin_username"
+ )
+ if not admin_username:
+ raise CannotBuild(
+ "fetch_service_auth_api_admin_username is not configured."
+ )
+ secret = _get_proxy_config("fetch_service_auth_api_admin_secret")
+ if not secret:
+ raise CannotBuild(
+ "fetch_service_auth_api_admin_secret is not configured."
+ )
+ url = _get_proxy_config("fetch_service_auth_api_endpoint")
+ if not secret:
+ raise CannotBuild(
+ "fetch_service_auth_api_endpoint is not configured."
+ )
+ timestamp = int(time.time())
+ proxy_username = "{build_id}-{timestamp}".format(
+ build_id=self.build.build_cookie, timestamp=timestamp
+ )
+ auth_string = f"{admin_username}:{secret}".strip()
+ auth_header = b"Basic " + base64.b64encode(auth_string.encode("ASCII"))
+
+ token = yield self._worker.process_pool.doWork(
+ RequestFetchServiceSessionCommand,
+ url=url,
+ auth_header=auth_header,
+ proxy_username=proxy_username,
+ )
+ return token
diff --git a/lib/lp/buildmaster/downloader.py b/lib/lp/buildmaster/downloader.py
index fc16852..cee308a 100644
--- a/lib/lp/buildmaster/downloader.py
+++ b/lib/lp/buildmaster/downloader.py
@@ -9,6 +9,7 @@ anything from the rest of Launchpad.
__all__ = [
"DownloadCommand",
+ "RequestFetchServiceSessionCommand",
"RequestProcess",
"RequestProxyTokenCommand",
]
@@ -37,6 +38,21 @@ class DownloadCommand(amp.Command):
}
+class RequestFetchServiceSessionCommand(amp.Command):
+ arguments = [
+ (b"url", amp.Unicode()),
+ (b"auth_header", amp.String()),
+ (b"proxy_username", amp.Unicode()),
+ ]
+ response = [
+ (b"id", amp.Unicode()),
+ (b"token", amp.Unicode()),
+ ]
+ errors = {
+ RequestException: b"REQUEST_ERROR",
+ }
+
+
class RequestProxyTokenCommand(amp.Command):
arguments = [
(b"url", amp.Unicode()),
@@ -94,3 +110,24 @@ class RequestProcess(AMPChild):
)
response.raise_for_status()
return response.json()
+
+ @RequestFetchServiceSessionCommand.responder
+ def requestFetchServiceSessionCommand(
+ self, url, auth_header, proxy_username
+ ):
+ with Session() as session:
+ session.trust_env = False
+ # XXX pelpsi 2024-02-28: should take the same
+ # json body? From ST108 we should provide
+ # {
+ # "timeout": <int>, // session timeout in seconds
+ # "policy": <string> // "strict" or "permissive"
+ # }
+ response = session.post(
+ url,
+ headers={"Authorization": auth_header},
+ json={"username": proxy_username},
+ )
+ print(" CI SIAMO???? ")
+ response.raise_for_status()
+ return response.json()
diff --git a/lib/lp/buildmaster/tests/fetchservice.py b/lib/lp/buildmaster/tests/fetchservice.py
new file mode 100644
index 0000000..9a12933
--- /dev/null
+++ b/lib/lp/buildmaster/tests/fetchservice.py
@@ -0,0 +1,119 @@
+# Copyright 2015-2019 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Fixtures for dealing with the build time HTTP proxy."""
+
+import json
+import uuid
+from textwrap import dedent
+from urllib.parse import urlsplit
+
+import fixtures
+from testtools.matchers import Equals, HasLength, MatchesStructure
+from twisted.internet import defer, endpoints, reactor
+from twisted.python.compat import nativeString
+from twisted.web import resource, server
+
+from lp.services.config import config
+
+
+class FetchServiceAuthAPITokensResource(resource.Resource):
+ """A test tokens resource for the proxy authentication API."""
+
+ isLeaf = True
+
+ def __init__(self):
+ resource.Resource.__init__(self)
+ self.requests = []
+
+ def render_POST(self, request):
+ content = json.loads(request.content.read().decode("UTF-8"))
+ self.requests.append(
+ {
+ "method": request.method,
+ "uri": request.uri,
+ "headers": dict(request.requestHeaders.getAllRawHeaders()),
+ "json": content,
+ }
+ )
+ return json.dumps(
+ {
+ "id": "1",
+ "token": uuid.uuid4().hex,
+ }
+ ).encode("UTF-8")
+
+
+class InProcessFetchServiceAuthAPIFixture(fixtures.Fixture):
+ """A fixture that pretends to be the proxy authentication API.
+
+ Users of this fixture must call the `start` method, which returns a
+ `Deferred`, and arrange for that to get back to the reactor. This is
+ necessary because the basic fixture API does not allow `setUp` to return
+ anything. For example:
+
+ class TestSomething(TestCase):
+
+ run_tests_with = AsynchronousDeferredRunTest.make_factory(
+ timeout=30)
+
+ @defer.inlineCallbacks
+ def setUp(self):
+ super().setUp()
+ yield self.useFixture(
+ InProcessFetchServiceAuthAPIFixture()
+ ).start()
+ """
+
+ @defer.inlineCallbacks
+ def start(self):
+ root = resource.Resource()
+ self.tokens = FetchServiceAuthAPITokensResource()
+ root.putChild(b"tokens", self.tokens)
+ endpoint = endpoints.serverFromString(reactor, nativeString("tcp:0"))
+ site = server.Site(self.tokens)
+ self.addCleanup(site.stopFactory)
+ port = yield endpoint.listen(site)
+ self.addCleanup(port.stopListening)
+ config.push(
+ "in-process-fetch-service-api-fixture",
+ dedent(
+ """
+ [builddmaster]
+ fetch_service_auth_api_admin_secret: admin-secret
+ fetch_service_auth_api_admin_username: admin-launchpad.test
+ fetch_service_auth_api_endpoint: http://{host}:{port}/tokens
+ fetch_service_host: {host}
+ fetch_service_port: {port}
+ """
+ ).format(host=port.getHost().host, port=port.getHost().port),
+ )
+ self.addCleanup(config.pop, "in-process-fetch-service-api-fixture")
+
+
+class FetchServiceURLMatcher(MatchesStructure):
+ """Check that a string is a valid url for a builder proxy."""
+
+ def __init__(self):
+ super().__init__(
+ scheme=Equals("http"),
+ id=Equals(1),
+ token=HasLength(32),
+ hostname=Equals(config.builddmaster.fetch_service_host),
+ port=Equals(config.builddmaster.fetch_service_port),
+ path=Equals(""),
+ )
+
+ def match(self, matchee):
+ super().match(urlsplit(matchee))
+
+
+class RevocationEndpointMatcher(Equals):
+ """Check that a string is a valid endpoint for proxy token revocation."""
+
+ def __init__(self, session_id):
+ super().__init__(
+ "{}/session/{}".format(
+ config.builddmaster.fetch_service_auth_api_endpoint, session_id
+ )
+ )
diff --git a/lib/lp/services/config/schema-lazr.conf b/lib/lp/services/config/schema-lazr.conf
index ac68bd4..9e2a463 100644
--- a/lib/lp/services/config/schema-lazr.conf
+++ b/lib/lp/services/config/schema-lazr.conf
@@ -167,6 +167,23 @@ builder_proxy_host: none
# Builder http proxy port
builder_proxy_port: none
+# Admin secret for requesting sessions from the fetch service.
+# For a mojo deployed proxy service, the secret is generated and will reside in
+# /srv/mojo/${MOJO_PROJECT}/${SERIES}/${MOJO_WORKSPACE}/local/admin-api-secret
+fetch_service_auth_api_admin_secret: none
+
+# Admin username for fetch service.
+fetch_service_auth_api_admin_username: none
+
+# Endpoint for fetch service authentication service
+fetch_service_auth_api_endpoint: none
+
+# Fetch service http proxy host
+fetch_service_host: none
+
+# Fetch service http proxy port
+fetch_service_port: none
+
[canonical]
# datatype: boolean
show_tracebacks: False
@@ -1854,6 +1871,23 @@ builder_proxy_host: none
# Deprecated in favour of the same key in the builddmaster section.
builder_proxy_port: none
+# Admin secret for requesting sessions from the fetch service.
+# For a mojo deployed proxy service, the secret is generated and will reside in
+# /srv/mojo/${MOJO_PROJECT}/${SERIES}/${MOJO_WORKSPACE}/local/admin-api-secret
+fetch_service_auth_api_admin_secret: none
+
+# Admin username for fetch service.
+fetch_service_auth_api_admin_username: none
+
+# Endpoint for fetch service authentication service
+fetch_service_auth_api_endpoint: none
+
+# Fetch service http proxy host
+fetch_service_host: none
+
+# Fetch service http proxy port
+fetch_service_port: none
+
# Optional sources.list entry to send to build workers when doing snap
# package builds. Use this form so that the series is set:
# deb http://foo %(series)s main
diff --git a/lib/lp/snappy/model/snapbuildbehaviour.py b/lib/lp/snappy/model/snapbuildbehaviour.py
index 66ae7cc..58fdd5f 100644
--- a/lib/lp/snappy/model/snapbuildbehaviour.py
+++ b/lib/lp/snappy/model/snapbuildbehaviour.py
@@ -116,7 +116,9 @@ class SnapBuildBehaviour(BuilderProxyMixin, BuildFarmJobBehaviourBase):
"""
build: ISnapBuild = self.build
args: BuildArgs = yield super().extraBuildArgs(logger=logger)
- yield self.addProxyArgs(args, build.snap.allow_internet)
+ yield self.addProxyArgs(
+ args, build.snap.allow_internet, build.snap.use_fetch_service
+ )
args["name"] = build.snap.store_name or build.snap.name
channels = build.channels or {}
if "snapcraft" not in channels:
diff --git a/lib/lp/snappy/tests/test_snapbuildbehaviour.py b/lib/lp/snappy/tests/test_snapbuildbehaviour.py
index 5ec5804..45ee8c8 100644
--- a/lib/lp/snappy/tests/test_snapbuildbehaviour.py
+++ b/lib/lp/snappy/tests/test_snapbuildbehaviour.py
@@ -51,6 +51,9 @@ from lp.buildmaster.tests.builderproxy import (
ProxyURLMatcher,
RevocationEndpointMatcher,
)
+from lp.buildmaster.tests.fetchservice import (
+ InProcessFetchServiceAuthAPIFixture,
+)
from lp.buildmaster.tests.mock_workers import (
MockBuilder,
OkWorker,
@@ -72,6 +75,7 @@ from lp.services.statsd.tests import StatsMixin
from lp.services.webapp import canonical_url
from lp.snappy.interfaces.snap import (
SNAP_SNAPCRAFT_CHANNEL_FEATURE_FLAG,
+ SNAP_USE_FETCH_SERVICE_FEATURE_FLAG,
SnapBuildArchiveOwnerMismatch,
)
from lp.snappy.model.snapbuildbehaviour import (
@@ -259,6 +263,23 @@ class TestAsyncSnapBuildBehaviour(StatsMixin, TestSnapBuildBehaviourBase):
)
)
self.proxy_api = self.useFixture(InProcessProxyAuthAPIFixture())
+ self.session = {
+ "id": "1",
+ "token": uuid.uuid4().hex,
+ }
+ self.fetch_service_url = (
+ "http://{session_id}:{token}@{host}:{port}".format(
+ session_id=self.session["id"],
+ token=self.session["token"],
+ host=config.builddmaster.fetch_service_host,
+ port=config.builddmaster.fetch_service_port,
+ )
+ )
+ self.fetch_service_api = self.useFixture(
+ InProcessFetchServiceAuthAPIFixture()
+ )
+ yield self.fetch_service_api.start()
+ self.proxy_api = self.useFixture(InProcessProxyAuthAPIFixture())
yield self.proxy_api.start()
self.now = time.time()
self.useFixture(fixtures.MockPatch("time.time", return_value=self.now))
@@ -296,6 +317,85 @@ class TestAsyncSnapBuildBehaviour(StatsMixin, TestSnapBuildBehaviourBase):
)
@defer.inlineCallbacks
+ def test_requestFetchServiceSession_unconfigured(self):
+ self.pushConfig("builddmaster", fetch_service_host=None)
+ self.useFixture(
+ FeatureFixture({SNAP_USE_FETCH_SERVICE_FEATURE_FLAG: "on"})
+ )
+ snap = self.factory.makeSnap(use_fetch_service=True)
+ request = self.factory.makeSnapBuildRequest(snap=snap)
+ job = self.makeJob(snap=snap, build_request=request)
+ with dbuser(config.builddmaster.dbuser):
+ args = yield job.extraBuildArgs()
+ self.assertEqual([], self.fetch_service_api.tokens.requests)
+ self.assertNotIn("proxy_url", args)
+ self.assertNotIn("revocation_endpoint", args)
+
+ @defer.inlineCallbacks
+ def test_requestFetchServiceSession_no_secret(self):
+ self.pushConfig(
+ "builddmaster", fetch_service_auth_api_admin_secret=None
+ )
+ self.useFixture(
+ FeatureFixture({SNAP_USE_FETCH_SERVICE_FEATURE_FLAG: "on"})
+ )
+ snap = self.factory.makeSnap(use_fetch_service=True)
+ request = self.factory.makeSnapBuildRequest(snap=snap)
+ job = self.makeJob(snap=snap, build_request=request)
+ expected_exception_msg = (
+ "fetch_service_auth_api_admin_secret is not configured."
+ )
+ with ExpectedException(CannotBuild, expected_exception_msg):
+ yield job.extraBuildArgs()
+
+ @defer.inlineCallbacks
+ def test_requestFetchServiceSession(self):
+ self.useFixture(
+ FeatureFixture({SNAP_USE_FETCH_SERVICE_FEATURE_FLAG: "on"})
+ )
+ snap = self.factory.makeSnap(use_fetch_service=True)
+ request = self.factory.makeSnapBuildRequest(snap=snap)
+ job = self.makeJob(snap=snap, build_request=request)
+ yield job.extraBuildArgs()
+ expected_uri = urlsplit(
+ config.builddmaster.fetch_service_auth_api_endpoint
+ ).path.encode("UTF-8")
+ request_matcher = MatchesDict(
+ {
+ "method": Equals(b"POST"),
+ "uri": Equals(expected_uri),
+ "headers": ContainsDict(
+ {
+ b"Authorization": MatchesListwise(
+ [
+ Equals(
+ b"Basic "
+ + base64.b64encode(
+ b"admin-launchpad.test:admin-secret"
+ )
+ )
+ ]
+ ),
+ b"Content-Type": MatchesListwise(
+ [
+ Equals(b"application/json"),
+ ]
+ ),
+ }
+ ),
+ "json": MatchesDict(
+ {
+ "username": StartsWith(job.build.build_cookie + "-"),
+ }
+ ),
+ }
+ )
+ self.assertThat(
+ self.fetch_service_api.tokens.requests,
+ MatchesListwise([request_matcher]),
+ )
+
+ @defer.inlineCallbacks
def test_requestProxyToken_unconfigured(self):
self.pushConfig("builddmaster", builder_proxy_host=None)
branch = self.factory.makeBranch()
Follow ups
