launchpad-reviewers team mailing list archive

[Colin Watson <mp+437090@xxxxxxxxxxxxxxxxxx>]

Colin Watson has proposed merging ~cjwatson/lp-archive:charm into lp-archive:main with ~cjwatson/lp-archive:service-config-env as a prerequisite.

Commit message:
Add a Juju charm

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~cjwatson/lp-archive/+git/lp-archive/+merge/437090

This is based on the layers provided by lp:ols-charm-deps.

This somewhat relies on getting artifact tarballs into PS5's object storage service, which is currently blocked on https://portal.admin.canonical.com/C155938, but it's possible to test it locally without that by building the artifact yourself.
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/lp-archive:charm into lp-archive:main.

diff --git a/charm/.gitignore b/charm/.gitignore
new file mode 100644
index 0000000..4e727f8
--- /dev/null
+++ b/charm/.gitignore
@@ -0,0 +1,3 @@
+*.charm
+dist
+tmp
diff --git a/charm/Makefile b/charm/Makefile
new file mode 100644
index 0000000..20dec9c
--- /dev/null
+++ b/charm/Makefile
@@ -0,0 +1,96 @@
+# The charmcraft tool is shipped as a snap, so make sure it's on $PATH.
+export PATH := $(PATH):/snap/bin
+
+APP_NAME := lp-archive
+
+BUILDDIR := $(CURDIR)/dist
+TMPDIR := $(CURDIR)/tmp
+CHARM_LAYERS_DIR := $(TMPDIR)/deps/ols-layers/layer
+CHARM_INTERFACES_DIR := $(TMPDIR)/deps/ols-layers/interface
+
+CHARM_SERIES ?= 22.04
+ARCH := $(shell dpkg --print-architecture)
+charm_file = $(1)_ubuntu-$(CHARM_SERIES)-$(ARCH).charm
+
+BUILD_LABEL = $(shell git rev-parse HEAD)
+TARBALL = $(APP_NAME).tar.gz
+ASSET = ../build/$(BUILD_LABEL)/$(TARBALL)
+
+CHARMS := lp-archive
+
+all: ## alias to build
+all: build
+
+help: ## display this help message
+	@echo "Please use \`make <target>' where <target> is one of"
+	@grep '^[a-zA-Z]' $(MAKEFILE_LIST) | sort | awk -F ':.*?## ' 'NF==2 {printf "\033[36m  %-25s\033[0m %s\n", $$1, $$2}'
+
+$(BUILDDIR) $(TMPDIR):
+	@mkdir -p $@
+
+# We have to clone our dependencies by hand rather than letting charmcraft
+# do it, since some of them are in private repositories and charmcraft
+# doesn't have suitable credentials.
+CHARM_DEPS := $(CHARM_LAYERS_DIR)/.done $(CHARM_INTERFACES_DIR)/.done
+$(CHARM_DEPS): $(CURDIR)/dependencies.txt | $(TMPDIR)
+	@echo "Fetching dependencies..."
+	@mkdir -p $(TMPDIR)/deps
+	@cd $(TMPDIR)/deps && codetree $<
+	@touch $(CHARM_DEPS)
+
+build: ## build all the charms
+build: $(foreach charm,$(CHARMS),build-$(charm))
+
+build-lp-archive: ## build the lp-archive charm
+build-lp-archive: dist/$(call charm_file,lp-archive)
+
+dist/%_ubuntu-$(CHARM_SERIES)-$(ARCH).charm: $(CHARM_DEPS) | $(BUILDDIR)
+	echo "Building $*..."
+	rm -rf $*/tmp
+	cp -a tmp $*/tmp
+	cd $* && charmcraft pack
+	cp -a $*/$(call charm_file,$*) dist/
+	rm -rf $*/tmp
+
+clean: ## clean the build environment
+clean: $(foreach charm,$(CHARMS),clean-$(charm))
+	@find . -name \*.pyc -delete
+	@find . -depth -name '__pycache__' -exec rm -rf '{}' \;
+	@rm -f bundle.yaml
+	@rm -f layer/*/codetree-collect-info.yaml
+	@rm -rf $(BUILDDIR) $(TMPDIR)
+
+clean-%:
+	@echo "Cleaning $*..."
+	@cd $* && charmcraft clean
+	@rm -f dist/$(call charm_file,$*)
+
+bundle.yaml: ## create the bundle.yaml file from the bundle.yaml.in template
+bundle.yaml: bundle.yaml.in
+	sed \
+	    -e 's/%BUILD_LABEL%/$(BUILD_LABEL)/g' \
+	    bundle.yaml.in >bundle.yaml
+
+deploy: ## deploy the built charm
+deploy: build bundle.yaml
+	@echo "Deploying $(APP_NAME)..."
+	@juju deploy ./bundle.yaml
+
+payload: ## build an lp-archive tarball
+payload: $(ASSET)
+$(ASSET):
+	@echo "Building asset for $(BUILD_LABEL)..."
+	@$(MAKE) -C .. build-tarball
+
+
+setup-jenkaas: ## prepare a Jenkins-as-a-service container for charm building
+setup-jenkaas:
+	sudo systemctl stop snapd.socket
+	sudo systemctl stop snapd
+	echo SNAPPY_STORE_NO_CDN=1 | sudo tee -a /etc/environment >/dev/null
+	echo SNAPPY_TESTING=1 | sudo tee -a /etc/environment >/dev/null
+	sudo systemctl start snapd.socket
+	sudo snap install --classic charm
+
+.PHONY: $(foreach charm,$(CHARMS),build-$(charm))
+.PHONY: all build clean deploy payload setup-jenkaas
diff --git a/charm/dependencies.txt b/charm/dependencies.txt
new file mode 100644
index 0000000..a0a0973
--- /dev/null
+++ b/charm/dependencies.txt
@@ -0,0 +1,2 @@
+ols-layers		git+ssh://git.launchpad.net/~ubuntuone-pqm-team/ols-charm-deps/+git/ols-layers;revno=e5787897
+charm-wheels		git+ssh://git.launchpad.net/~ubuntuone-hackers/ols-charm-deps/+git/wheels;revno=fbf2c56c
diff --git a/charm/lp-archive/charmcraft.yaml b/charm/lp-archive/charmcraft.yaml
new file mode 100644
index 0000000..76b6fae
--- /dev/null
+++ b/charm/lp-archive/charmcraft.yaml
@@ -0,0 +1,20 @@
+type: charm
+bases:
+  - build-on:
+    - name: ubuntu
+      channel: "22.04"
+    run-on:
+    - name: ubuntu
+      channel: "22.04"
+parts:
+  lp-archive:
+    source: .
+    plugin: reactive
+    build-snaps: [charm/2.x/stable]
+    build-environment:
+      - CHARM_LAYERS_DIR: tmp/deps/ols-layers/layer
+      - CHARM_INTERFACES_DIR: tmp/deps/ols-layers/interface
+      - PIP_NO_INDEX: "true"
+      - PIP_FIND_LINKS: tmp/deps/charm-wheels
+    stage:
+      - -tmp
diff --git a/charm/lp-archive/config.yaml b/charm/lp-archive/config.yaml
new file mode 100644
index 0000000..e9a0ca6
--- /dev/null
+++ b/charm/lp-archive/config.yaml
@@ -0,0 +1,17 @@
+options:
+  archive_endpoint:
+    type: string
+    default: http://xmlrpc-private.launchpad.test:8087/archive
+    description: URL of private Launchpad XML-RPC archive endpoint.
+  layouts:
+    type: string
+    default: ""
+    description: >
+      YAML-encoded list of layouts offered by this deployment.  Each layout is
+      a dictionary containing "host" and "purpose" keys, where "purpose" is
+      "primary" or "ppa".  For example: [{"host": "snapshot.ubuntu.test:8000",
+      "purpose": "primary"}].
+  log_hosts_allow:
+    type: string
+    default: ""
+    description: Hosts that should be allowed to rsync logs.
diff --git a/charm/lp-archive/layer.yaml b/charm/lp-archive/layer.yaml
new file mode 100644
index 0000000..11d23f2
--- /dev/null
+++ b/charm/lp-archive/layer.yaml
@@ -0,0 +1,18 @@
+includes:
+  - layer:ols-wsgi
+  - interface:memcache
+options:
+  apt:
+    packages:
+      - libmemcached11
+  ols:
+    service_name: lp-archive
+    config_filename: config.toml
+    user: launchpad
+    tarball_payload: true
+    symlink_switch_payload: true
+  ols-http:
+    port: 8000
+  ols-wsgi:
+    app: lp_archive
+repo: https://git.launchpad.net/lp-archive
diff --git a/charm/lp-archive/metadata.yaml b/charm/lp-archive/metadata.yaml
new file mode 100644
index 0000000..0774d2f
--- /dev/null
+++ b/charm/lp-archive/metadata.yaml
@@ -0,0 +1,19 @@
+name: lp-archive
+display-name: lp-archive
+summary: Launchpad archive frontend service
+maintainer: Colin Watson <cjwatson@xxxxxxxxxxxxx>
+description: >
+  A frontend service to serve Launchpad archives (such as the primary Ubuntu
+  archive or PPAs) over HTTP without the need for local storage of the
+  archive files.
+series:
+  - jammy
+subordinate: false
+requires:
+  memcache:
+    interface: memcache
+resources:
+  lp-archive:
+    type: file
+    filename: lp-archive.tar.gz
+    description: lp-archive code
diff --git a/charm/lp-archive/reactive/lp-archive.py b/charm/lp-archive/reactive/lp-archive.py
new file mode 100644
index 0000000..28c8efe
--- /dev/null
+++ b/charm/lp-archive/reactive/lp-archive.py
@@ -0,0 +1,66 @@
+# Copyright 2023 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+import yaml
+from charmhelpers.core import hookenv, host, templating
+from charms.reactive import hook, remove_state, set_state, when, when_not
+from ols import base, http
+
+
+def configure_rsync():
+    config = hookenv.config()
+    if config["log_hosts_allow"]:
+        rsync_config = dict(config)
+        rsync_config["base_dir"] = base.base_dir()
+        templating.render(
+            "lp-archive-rsync.j2",
+            "/etc/rsync-juju.d/010-lp-archive.conf",
+            rsync_config,
+            perms=0o644,
+        )
+        if not host.service_restart("rsync"):
+            raise RuntimeError("Failed to restart rsync")
+
+
+@when("ols.configured", "memcache.available")
+@when_not("service.configured")
+def configure(memcache):
+    config = dict(hookenv.config())
+    config["memcache_servers"] = sorted(
+        f"{host}:{port}" for host, port in memcache.memcache_hosts_ports()
+    )
+    config["layouts"] = yaml.safe_load(config["layouts"])
+
+    hookenv.log("Writing service config.")
+    templating.render(
+        "config.toml.j2",
+        base.service_config_path(),
+        config,
+        owner="root",
+        group=base.user(),
+        perms=0o440,
+    )
+
+    configure_rsync()
+
+    set_state("service.configured")
+
+
+@when("service.configured", "ols.wsgi.configured")
+def check_is_running():
+    if http.is_listening(url_path="/_status/check"):
+        hookenv.status_set("active", "Ready")
+    else:
+        hookenv.status_set("blocked", "Service not running, check logs")
+
+
+@when("config.changed.build_label")
+def build_label_changed():
+    remove_state("ols.service.installed")
+    remove_state("ols.configured")
+    remove_state("service.configured")
+
+
+@hook("{requires:memcache}-relation-changed")
+def memcache_relation_changed(memcache):
+    remove_state("service.configured")
diff --git a/charm/lp-archive/templates/config.toml.j2 b/charm/lp-archive/templates/config.toml.j2
new file mode 100644
index 0000000..5512086
--- /dev/null
+++ b/charm/lp-archive/templates/config.toml.j2
@@ -0,0 +1,12 @@
+ARCHIVE_ENDPOINT = "{{ archive_endpoint }}"
+CACHE_MEMCACHED_SERVERS = [{% for server in memcache_servers %}"{{ server }}"{% if not loop.last %}, {% endif %}{% endfor %}]
+CACHE_TYPE = "MemcachedCache"
+{%- if layouts %}
+{%- for layout in layouts %}
+
+[[LAYOUTS]]
+host = "{{ layout["host"] }}"
+purpose = "{{ layout["purpose"] }}"
+{%- endfor %}
+{%- endif %}
+
diff --git a/charm/lp-archive/templates/lp-archive-rsync.j2 b/charm/lp-archive/templates/lp-archive-rsync.j2
new file mode 100644
index 0000000..52c519d
--- /dev/null
+++ b/charm/lp-archive/templates/lp-archive-rsync.j2
@@ -0,0 +1,8 @@
+
+[lp-archive-logs]
+  path = {{ base_dir }}/logs
+  comment = LP Archive Logs
+  list = false
+  read only = true
+  hosts allow = {{ log_hosts_allow }}
+
diff --git a/charm/packages.txt b/charm/packages.txt
new file mode 100644
index 0000000..c33fce5
--- /dev/null
+++ b/charm/packages.txt
@@ -0,0 +1,3 @@
+flake8
+python-codetree
+squashfuse
diff --git a/tox.ini b/tox.ini
index c054870..eac8482 100644
--- a/tox.ini
+++ b/tox.ini
@@ -13,7 +13,7 @@ skip_missing_interpreters =
 description =
     run test suite
 commands =
-    pytest {posargs}
+    pytest {posargs:tests}
 deps =
     -r requirements.txt
     .[test]
@@ -62,6 +62,6 @@ deps =
     .[test]
 commands =
     coverage erase
-    coverage run -m pytest
+    coverage run -m pytest tests
     coverage html
     coverage report -m  --fail-under=100