launchpad-reviewers team mailing list archive

[Colin Watson <mp+435882@xxxxxxxxxxxxxxxxxx>]

Colin Watson has proposed merging ~cjwatson/lp-archive:build-machinery into lp-archive:main.

Commit message:
Add some build machinery and a basic README

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~cjwatson/lp-archive/+git/lp-archive/+merge/435882

This integrates with our usual artifact building setup that we use to publish builds to Swift so that we can deploy them using Juju.  Based on this, it also adds some minimal documentation explaining how Launchpad developers can run the service locally.
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/lp-archive:build-machinery into lp-archive:main.

diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..b3ae251
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,89 @@
+SERVICE_PACKAGE = lp_archive
+ENV = $(CURDIR)/env
+WHEELS = $(CURDIR)/wheels
+PYTHON3 = $(ENV)/bin/python3
+PIP = $(PYTHON3) -m pip
+TMPDIR = $(CURDIR)/tmp
+BIND ?= 0.0.0.0:8000
+GUNICORN = $(ENV)/bin/talisker
+
+# development config
+export SERVICE_CONFIG = $(CURDIR)/service.conf
+
+GITREF ?= $(shell git symbolic-ref HEAD)
+GITHASH ?= $(shell git rev-parse $(or $(TAG),$(GITREF)))
+ifndef GITHASH
+    $(error "There was no reference '$(GITREF)' found.")
+endif
+GITTAGS ?= $(shell git tag)
+
+DEPENDENCY_REPO ?= lp:~launchpad/lp-archive/+git/dependencies
+DEPENDENCY_DIR ?= $(TMPDIR)/dependencies
+
+# Create archives in labelled directories (e.g.
+# <rev-id>/$(PROJECT_NAME).tar.gz)
+TARBALL_BUILD_LABEL ?= $(shell git rev-parse HEAD)
+TARBALL_FILE_NAME = lp-archive.tar.gz
+TARBALL_BUILDS_DIR ?= build
+TARBALL_BUILD_DIR = $(TARBALL_BUILDS_DIR)/$(TARBALL_BUILD_LABEL)
+TARBALL_BUILD_PATH = $(TARBALL_BUILD_DIR)/$(TARBALL_FILE_NAME)
+
+SWIFT_CONTAINER_NAME ?= lp-archive-builds
+# This must match the object path used by fetch_payload in the ols charm
+# layer.
+SWIFT_OBJECT_PATH = \
+	lp-archive-builds/$(TARBALL_BUILD_LABEL)/$(TARBALL_FILE_NAME)
+
+
+$(ENV): | $(DEPENDENCY_DIR)
+	/usr/bin/virtualenv $(ENV) --python=python3
+	$(PIP) install -f $(DEPENDENCY_DIR) --no-index -r requirements.txt -e .
+	git rev-parse HEAD >version-info.txt
+	@touch $@
+
+$(DEPENDENCY_DIR):
+	git clone $(DEPENDENCY_REPO) $(DEPENDENCY_DIR)
+
+.PHONY: update-dependencies
+update-dependencies: $(DEPENDENCY_DIR)
+	cd $(DEPENDENCY_DIR) && git pull $(DEPENDENCY_REPO)
+
+.PHONY: bootstrap
+bootstrap: $(ENV)
+
+.PHONY: clean
+clean:
+	rm -rf $(ENV) $(WHEELS) version-info.txt
+	rm -rf $(TMPDIR)
+	find -name '__pycache__' -print0 | xargs -0 rm -rf
+	find -name '*.~*' -delete
+
+.PHONY: run
+run: $(ENV)
+	DEVEL=1 $(GUNICORN) --reload --log-level debug -b $(BIND) $(SERVICE_PACKAGE)
+
+# XXX cjwatson 2023-01-09: limit to only interesting files
+.PHONY: build-tarball
+build-tarball:
+	@echo "Creating deployment tarball at $(TARBALL_BUILD_PATH)"
+	rm -rf $(ENV)
+	$(MAKE) $(ENV)
+	$(PIP) wheel -f $(DEPENDENCY_DIR) --no-index \
+		-w $(WHEELS) -r requirements.txt
+	mkdir -p $(TARBALL_BUILD_DIR)
+	tar -czf $(TARBALL_BUILD_PATH) \
+		--exclude-vcs \
+		--exclude .tox \
+		--exclude build \
+		--exclude charm \
+		--exclude dist \
+		--exclude env \
+		--exclude tmp \
+		./
+
+.PHONY: publish-tarball
+publish-tarball: build-tarball
+	[ ! -e ~/.config/swift/lp-archive ] || . ~/.config/swift/lp-archive; \
+	./publish-to-swift --debug \
+		$(SWIFT_CONTAINER_NAME) $(SWIFT_OBJECT_PATH) \
+		$(TARBALL_BUILD_PATH)
diff --git a/README.rst b/README.rst
new file mode 100644
index 0000000..72753c7
--- /dev/null
+++ b/README.rst
@@ -0,0 +1,47 @@
+lp-archive
+==========
+
+A frontend service to serve Launchpad archives (such as the primary Ubuntu
+archive or PPAs) over HTTP without the need for local storage of the archive
+files.
+
+
+Development environment
+-----------------------
+
+To run tests, use ``tox``.
+
+Running a local development server requires some setup and can only be done
+if you have privileged access to a suitable Launchpad environment.
+Launchpad staff can start by setting up an SSH tunnel to the "dogfood"
+environment, in a separate terminal::
+
+    $ ssh -L 8097:xmlrpc.dogfood.lp.internal:8097 labbu.canonical.com
+
+Add ``127.0.0.1 xmlrpc.dogfood.lp.internal`` to ``/etc/hosts`` on your
+development system to fool it into connecting via this tunnel.
+
+You'll then need a ``config.toml`` file telling ``lp-archive`` what URL
+layout to expose.  For example:
+
+.. code-block:: toml
+
+    ARCHIVE_ENDPOINT = "http://xmlrpc.dogfood.lp.internal:8097/archive";
+    CACHE_TYPE = "SimpleCache"
+
+    [[LAYOUTS]]
+    host = "snapshot.ubuntu.test:8000"
+    purpose = "primary"
+
+    [[LAYOUTS]]
+    host = "snapshot.ppa.test:8000"
+    purpose = "ppa"
+
+Start the service::
+
+    $ make run
+
+You should now be able to make HTTP requests.  The host name is significant,
+so the simplest way is to use ``curl --resolve``.  For example::
+
+    $ curl --resolve snapshot.ubuntu.test:8000:127.0.0.1 -L http://snapshot.ubuntu.test:8000/ubuntu/dists/focal/InRelease