launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #29311
[Merge] ~cjwatson/lp-archive:translate-paths into lp-archive:main
Colin Watson has proposed merging ~cjwatson/lp-archive:translate-paths into lp-archive:main.
Commit message:
Add archive endpoints
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/lp-archive/+git/lp-archive/+merge/431569
If configured with a suitable Launchpad XML-RPC endpoint, this can serve apt archives based on publishing records and the librarian without needing access to the published archive on a local file system.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/lp-archive:translate-paths into lp-archive:main.
diff --git a/.gitignore b/.gitignore
index af6421f..2c2f17b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,7 @@
/.python-version
/.tox
/build
+/config.toml
/env
/htmlcov
/tmp
diff --git a/.mypy.ini b/.mypy.ini
new file mode 100644
index 0000000..07a8171
--- /dev/null
+++ b/.mypy.ini
@@ -0,0 +1,5 @@
+[mypy]
+python_version = 3.10
+
+[mypy-tomllib.*]
+ignore_missing_imports = true
diff --git a/lp_archive/__init__.py b/lp_archive/__init__.py
index 63a22b1..83c12ec 100644
--- a/lp_archive/__init__.py
+++ b/lp_archive/__init__.py
@@ -3,16 +3,25 @@
"""The Launchpad archive service."""
+try:
+ import tomllib
+except ModuleNotFoundError:
+ import tomli as tomllib # type: ignore
from typing import Any
from flask import Flask
-from lp_archive import root
+from lp_archive import archive, root, routing
def create_app(test_config: dict[str, Any] | None = None) -> Flask:
app = Flask(__name__)
- if test_config is not None:
+ if test_config is None:
+ with open("config.toml", "rb") as f:
+ app.config.from_mapping(tomllib.load(f))
+ else:
app.config.from_mapping(test_config)
+ app.url_map.converters["archive"] = routing.ArchiveConverter
app.register_blueprint(root.bp)
+ app.register_blueprint(archive.bp)
return app
diff --git a/lp_archive/archive.py b/lp_archive/archive.py
new file mode 100644
index 0000000..5b2a683
--- /dev/null
+++ b/lp_archive/archive.py
@@ -0,0 +1,79 @@
+# Copyright 2022 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""The main archive view."""
+
+from xmlrpc.client import Fault, ServerProxy
+
+from flask import Blueprint, current_app, g, request
+from werkzeug.datastructures import WWWAuthenticate
+from werkzeug.exceptions import Unauthorized
+
+bp = Blueprint("archive", __name__)
+
+
+def get_archive_proxy() -> ServerProxy:
+ archive_proxy = getattr(g, "archive_proxy", None)
+ if archive_proxy is None:
+ archive_proxy = ServerProxy(
+ current_app.config["ARCHIVE_ENDPOINT"], allow_none=True
+ )
+ g.archive_proxy = archive_proxy
+ return archive_proxy
+
+
+def check_auth(archive: str) -> None:
+ """Check whether the current request may access an archive.
+
+ If unauthorized, raises a suitable exception.
+ """
+ # Ideally we'd use a Flask extension for this rather than rolling most
+ # of the HTTP Basic Authentication logic for ourselves, but nothing
+ # seems to be quite suitable. In particular, `flask-httpauth` doesn't
+ # currently support passing additional data from the route through to
+ # `verify_password`.
+ if request.authorization is None:
+ username = None
+ password = None
+ log_prefix = f"<anonymous>@{archive}"
+ else:
+ username = request.authorization.username
+ password = request.authorization.password
+ log_prefix = f"{username}@{archive}"
+ try:
+ # XXX cjwatson 2022-10-14: We should cache positive responses (maybe
+ # using `flask-caching`) for a while to reduce database load.
+ get_archive_proxy().checkArchiveAuthToken(archive, username, password)
+ except Fault as e:
+ if e.faultCode == 410: # Unauthorized
+ current_app.logger.info("%s: Password does not match.", log_prefix)
+ else:
+ # Interpret any other fault as NotFound (320).
+ current_app.logger.info("%s: %s", log_prefix, e.faultString)
+ basic = WWWAuthenticate()
+ basic.set_basic()
+ raise Unauthorized(www_authenticate=basic)
+ else:
+ current_app.logger.info("%s: Authorized.", log_prefix)
+
+
+# The exact details of the URLs used here should be regarded as a proof of
+# concept for now.
+@bp.route("/<archive:archive>/<path:path>")
+def translate(archive: str, path: str) -> tuple[str, int, dict[str, str]]:
+ check_auth(archive)
+ try:
+ url = get_archive_proxy().translatePath(archive, path)
+ except Fault as f:
+ if f.faultCode == 320: # NotFound
+ return "Not found", 404, {"Content-Type": "text/plain"}
+ else:
+ return "Internal server error", 500, {"Content-Type": "text/plain"}
+ assert isinstance(url, str)
+ return "", 307, {"Location": url}
+
+
+@bp.after_request
+def add_headers(response):
+ response.headers["Vary"] = "Authorization"
+ return response
diff --git a/lp_archive/routing.py b/lp_archive/routing.py
new file mode 100644
index 0000000..a3be94d
--- /dev/null
+++ b/lp_archive/routing.py
@@ -0,0 +1,23 @@
+# Copyright 2022 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Routing helpers."""
+
+from werkzeug.routing import BaseConverter
+
+
+class ArchiveConverter(BaseConverter):
+ """Match an archive reference.
+
+ See `lp.soyuz.model.archive.Archive.reference` in Launchpad.
+ """
+
+ # This doesn't currently support the partner/copy archive reference
+ # syntax (distribution/archive), since it's hard to avoid that being
+ # ambiguous when parsing URLs (compare with the primary archive
+ # reference syntax).
+ #
+ # PPA: ~[^/]+/[^/]+/[^/]+ (~owner/distribution/archive)
+ # Primary: [^~+][^/]* (distribution)
+ regex = r"~[^/]+/[^/]+/[^/]+|[^~+][^/]*"
+ part_isolating = False
diff --git a/requirements.in b/requirements.in
index e3e9a71..da656c4 100644
--- a/requirements.in
+++ b/requirements.in
@@ -1 +1,2 @@
Flask
+tomli; python_version < "3.11"
diff --git a/requirements.txt b/requirements.txt
index 27acd36..73d76c5 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -16,5 +16,7 @@ markupsafe==2.1.1
# via
# jinja2
# werkzeug
+tomli==2.0.1 ; python_version < "3.11"
+ # via -r requirements.in
werkzeug==2.2.2
# via flask
diff --git a/setup.cfg b/setup.cfg
index 8410614..f54019b 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -18,6 +18,7 @@ classifiers =
packages = find:
install_requires =
Flask
+ tomli;python_version < "3.11"
python_requires = >=3.10
[options.extras_require]
diff --git a/tests/test_archive.py b/tests/test_archive.py
new file mode 100644
index 0000000..22bd8e1
--- /dev/null
+++ b/tests/test_archive.py
@@ -0,0 +1,121 @@
+# Copyright 2022 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from threading import Thread
+from typing import Any
+from xmlrpc.client import Fault
+from xmlrpc.server import SimpleXMLRPCServer
+
+import pytest
+
+
+class ArchiveXMLRPCServer(SimpleXMLRPCServer):
+
+ path_map = {"dists/focal/InRelease": "http://librarian.example.org/1"}
+
+ def __init__(self, *args: Any, **kwargs: Any) -> None:
+ super().__init__(*args, **kwargs)
+ self.call_log = []
+ self.register_function(
+ self.check_archive_auth_token, name="checkArchiveAuthToken"
+ )
+ self.register_function(self.translate_path, name="translatePath")
+
+ def check_archive_auth_token(self, archive, username, password):
+ self.call_log.append(
+ ("checkArchiveAuthToken", archive, username, password)
+ )
+ if archive.endswith("/private"):
+ raise Fault(410, "Authorization required")
+ elif archive.endswith("/nonexistent"):
+ raise Fault(320, "Not found")
+ else:
+ return True
+
+ def translate_path(self, archive, path):
+ # See `lp.xmlrpc.faults` in Launchpad for fault codes.
+ self.call_log.append(("translatePath", archive, path))
+ if path == "oops":
+ raise Fault(380, "Oops")
+ elif path in self.path_map:
+ return self.path_map[path]
+ else:
+ raise Fault(320, "Not found")
+
+
+@pytest.fixture
+def archive_proxy(app):
+ with ArchiveXMLRPCServer(("127.0.0.1", 0)) as server:
+ host, port = server.server_address
+ app.config.update({"ARCHIVE_ENDPOINT": f"http://{host}:{port}"})
+ thread = Thread(target=server.serve_forever)
+ thread.start()
+ yield server
+ server.shutdown()
+ thread.join()
+
+
+def test_auth_failed(client, archive_proxy):
+ response = client.get(
+ "/~user/ubuntu/private/dists/focal/InRelease", auth=("user", "secret")
+ )
+ assert response.status_code == 401
+ assert (
+ response.headers["WWW-Authenticate"]
+ == 'Basic realm="authentication required"'
+ )
+ assert response.headers["Vary"] == "Authorization"
+ assert archive_proxy.call_log == [
+ ("checkArchiveAuthToken", "~user/ubuntu/private", "user", "secret")
+ ]
+
+
+def test_auth_not_found(client, archive_proxy):
+ response = client.get(
+ "/~user/ubuntu/nonexistent/dists/focal/InRelease",
+ auth=("user", "secret"),
+ )
+ assert response.status_code == 401
+ assert (
+ response.headers["WWW-Authenticate"]
+ == 'Basic realm="authentication required"'
+ )
+ assert response.headers["Vary"] == "Authorization"
+ assert archive_proxy.call_log == [
+ ("checkArchiveAuthToken", "~user/ubuntu/nonexistent", "user", "secret")
+ ]
+
+
+def test_translate(client, archive_proxy):
+ response = client.get("/ubuntu/dists/focal/InRelease")
+ assert response.status_code == 307
+ assert response.headers["Location"] == "http://librarian.example.org/1";
+ assert response.headers["Vary"] == "Authorization"
+ assert archive_proxy.call_log == [
+ ("checkArchiveAuthToken", "ubuntu", None, None),
+ ("translatePath", "ubuntu", "dists/focal/InRelease"),
+ ]
+
+
+def test_translate_not_found(client, archive_proxy):
+ response = client.get("/ubuntu/nonexistent")
+ assert response.status_code == 404
+ assert response.headers["Content-Type"] == "text/plain"
+ assert response.headers["Vary"] == "Authorization"
+ assert response.data == b"Not found"
+ assert archive_proxy.call_log == [
+ ("checkArchiveAuthToken", "ubuntu", None, None),
+ ("translatePath", "ubuntu", "nonexistent"),
+ ]
+
+
+def test_translate_oops(client, archive_proxy):
+ response = client.get("/ubuntu/oops")
+ assert response.status_code == 500
+ assert response.headers["Content-Type"] == "text/plain"
+ assert response.headers["Vary"] == "Authorization"
+ assert response.data == b"Internal server error"
+ assert archive_proxy.call_log == [
+ ("checkArchiveAuthToken", "ubuntu", None, None),
+ ("translatePath", "ubuntu", "oops"),
+ ]
diff --git a/tests/test_routing.py b/tests/test_routing.py
new file mode 100644
index 0000000..e11ae4e
--- /dev/null
+++ b/tests/test_routing.py
@@ -0,0 +1,41 @@
+# Copyright 2022 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from flask import url_for
+
+
+def test_primary(app, client):
+ @app.route("/+test/<archive:archive>")
+ def index(archive):
+ return archive
+
+ response = client.get("/+test/ubuntu")
+ assert response.status_code == 200
+ assert response.data == b"ubuntu"
+
+ with app.test_request_context():
+ assert url_for("index", archive="ubuntu") == "/+test/ubuntu"
+
+
+def test_ppa(app, client):
+ @app.route("/+test/<archive:archive>")
+ def index(archive):
+ return archive
+
+ response = client.get("/+test/~owner/ubuntu/ppa")
+ assert response.status_code == 200
+ assert response.data == b"~owner/ubuntu/ppa"
+
+ with app.test_request_context():
+ assert (
+ url_for("index", archive="~owner/ubuntu/ppa")
+ == "/+test/~owner/ubuntu/ppa"
+ )
+
+
+def test_invalid_archive(app, client):
+ @app.route("/+test/<archive:archive>")
+ def index(archive): # pragma: no cover
+ return archive
+
+ assert client.get("/+test/~owner/ubuntu").status_code == 404
