launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #24401
[Merge] ~cjwatson/lp-signing:charm into lp-signing:master
Colin Watson has proposed merging ~cjwatson/lp-signing:charm into lp-signing:master with ~cjwatson/lp-signing:generate-key-pair-path-options as a prerequisite.
Commit message:
Add a Juju charm
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/lp-signing/+git/lp-signing/+merge/380041
This is based on the layers provided by lp:ols-charm-deps.
Building the tarball requires a few extra tweaks: in particular, we need to add suitable versions of pip, setuptools, and wheel to requirements.txt in order that appropriate versions of these packages will be built into the deployment artifact and used when creating the virtualenv.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/lp-signing:charm into lp-signing:master.
diff --git a/.gitignore b/.gitignore
index 8d2da37..4653353 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,8 +2,10 @@
/.coverage
/.python-version
/.tox
+/build
/dev-db
/env
/htmlcov
/tmp
+/wheels
__pycache__
diff --git a/Makefile b/Makefile
index f54a304..17013a5 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,6 @@
SERVICE_PACKAGE = lp_signing
ENV = $(CURDIR)/env
+WHEELS = $(CURDIR)/wheels
PYTHON3 = $(ENV)/bin/python3
PIP = $(PYTHON3) -m pip
FLAKE8 = $(ENV)/bin/flake8
@@ -21,6 +22,14 @@ GITTAGS ?= $(shell git tag)
DEPENDENCY_REPO ?= lp:~launchpad/lp-signing/+git/dependencies
DEPENDENCY_DIR ?= $(TMPDIR)/dependencies
+# Create archives in labelled directories (e.g.
+# <rev-id>/$(PROJECT_NAME).tar.gz)
+TARBALL_BUILD_LABEL ?= $(shell git rev-parse HEAD)
+TARBALL_FILE_NAME = lp-signing.tar.gz
+TARBALL_BUILDS_DIR ?= build
+TARBALL_BUILD_DIR = $(TARBALL_BUILDS_DIR)/$(TARBALL_BUILD_LABEL)
+TARBALL_BUILD_PATH = $(TARBALL_BUILD_DIR)/$(TARBALL_FILE_NAME)
+
$(ENV)/prod: | $(DEPENDENCY_DIR)
virtualenv $(ENV) --python=python3
@@ -38,7 +47,7 @@ $(ENV)/dev: $(ENV)/prod
$(PIP) install -f $(DEPENDENCY_DIR) -c requirements-dev.txt -e '.[test]'
@touch $@
-bootstrap: $(ENV)/prod
+bootstrap build: $(ENV)/prod
# DEVEL=1 is to indicate to 'talisker' we're under devel, which in the
# end will make 'acceptable' to validate endpoints' output
@@ -57,7 +66,7 @@ coverage: $(ENV)/dev
$(PYTHON3) -m coverage html
clean:
- rm -rf $(ENV)
+ rm -rf $(ENV) $(WHEELS)
rm -rf $(TMPDIR)
rm -rf .coverage htmlcov
find -name '__pycache__' -print0 | xargs -0 rm -rf
@@ -66,6 +75,25 @@ clean:
run: $(ENV)/prod
DEVEL=1 $(GUNICORN) --reload --log-level debug -b $(BIND) $(SERVICE_PACKAGE).webapp:app
+# XXX cjwatson 2020-01-20: limit to only interesting files
+build-tarball:
+ @echo "Creating deployment tarball at $(TARBALL_BUILD_PATH)"
+ rm -rf $(ENV)
+ $(MAKE) $(ENV)/prod
+ $(PIP) wheel -f $(DEPENDENCY_DIR) --no-index \
+ -w $(WHEELS) -r requirements.txt
+ mkdir -p $(TARBALL_BUILD_DIR)
+ tar -czf $(TARBALL_BUILD_PATH) \
+ --exclude-vcs \
+ --exclude .tox \
+ --exclude build \
+ --exclude charm \
+ --exclude dev-db \
+ --exclude dist \
+ --exclude env \
+ --exclude tmp \
+ ./
+
include Makefile.db
.PHONY: dbshell update-dependencies test clean coverage run migrate reset-db setup-db start-db stop-db
diff --git a/charm/.gitignore b/charm/.gitignore
new file mode 100644
index 0000000..04ae82c
--- /dev/null
+++ b/charm/.gitignore
@@ -0,0 +1,3 @@
+bundle.yaml
+dist
+tmp
diff --git a/charm/Makefile b/charm/Makefile
new file mode 100644
index 0000000..a9218f2
--- /dev/null
+++ b/charm/Makefile
@@ -0,0 +1,130 @@
+# The charm tool is shipped as a snap, so make sure it's on $PATH.
+export PATH := $(PATH):/snap/bin
+
+APP_NAME := lp-signing
+
+BUILDDIR := $(CURDIR)/dist
+TMPDIR := $(CURDIR)/tmp
+export LAYER_PATH := $(TMPDIR)/deps/ols-layers/layer
+export INTERFACE_PATH := $(TMPDIR)/deps/ols-layers/interface
+CHARM_WHEELS_DIR := $(TMPDIR)/deps/charm-wheels
+
+BUILD_LABEL = $(shell git rev-parse HEAD)
+TARBALL = $(APP_NAME).tar.gz
+ASSET = ../build/$(BUILD_LABEL)/$(TARBALL)
+
+CHARMS := lp-signing
+
+PUBLISH_REPO_PREFIX := lp:~launchpad/lp-signing/+git/charm-build-
+PUBLISHDIR := $(BUILDDIR)/publish
+# We may need to force username and email when publishing, because git may
+# not be able to autodetect this in automatic build environments.
+DOMAIN ?= $(shell hostname -f)
+GIT_USERNAME = $(shell git config --get user.name || echo $(USER))
+GIT_EMAIL = $(shell git config --get user.email || echo $(USER)@$(DOMAIN))
+
+all: build lint
+
+$(BUILDDIR) $(TMPDIR) $(PUBLISHDIR):
+ @mkdir -p $@
+
+CHARM_DEPS := $(LAYER_PATH)/.done $(INTERFACE_PATH)/.done
+$(CHARM_DEPS): $(CURDIR)/dependencies.txt | $(TMPDIR)
+ @echo "Fetching dependencies..."
+ @mkdir -p $(TMPDIR)/deps
+ @cd $(TMPDIR)/deps && codetree $<
+ @touch $(CHARM_DEPS)
+
+build: $(foreach charm,$(CHARMS),build-$(charm))
+
+build-lp-signing: dist/.built-lp-signing
+
+dist/.built-%: $(CHARM_DEPS) | $(BUILDDIR)
+ @echo "Building $*..."
+ @cd $* && \
+ PIP_NO_INDEX=true PIP_FIND_LINKS=$(CHARM_WHEELS_DIR) \
+ charm build -o $(BUILDDIR)
+ @touch $@
+
+clean-%:
+ @echo "Cleaning $*..."
+ @rm -rf dist/.built-$* dist/builds/$* $(PUBLISHDIR)/$*
+
+tmp/signing.launchpad.test.crt: | $(TMPDIR)
+ openssl req -new -nodes -keyout tmp/signing.launchpad.test.key \
+ -out tmp/signing.launchpad.test.csr \
+ -subj '/CN=signing.launchpad.test'
+ openssl x509 -req -days 365 -in tmp/signing.launchpad.test.csr \
+ -signkey tmp/signing.launchpad.test.key -out $@
+
+tmp/service-private-key: | $(TMPDIR)
+ @$(MAKE) -C ..
+ ../env/bin/lp-signing generate-key-pair \
+ --private-key-path tmp/service-private-key \
+ --public-key-path tmp/service-public-key
+
+bundle.yaml: bundle.yaml.in tmp/signing.launchpad.test.crt tmp/service-private-key
+ sed -e 's/%BUILD_LABEL%/$(BUILD_LABEL)/g' \
+ -e "s/%SSL_KEY%/$$(base64 -w 0 <tmp/signing.launchpad.test.key)/g" \
+ -e "s/%SSL_CERT%/$$(base64 -w 0 <tmp/signing.launchpad.test.crt)/g" \
+ -e "s/%SERVICE_PRIVATE_KEY%/$$(cat tmp/service-private-key)/g" \
+ bundle.yaml.in >bundle.yaml
+
+deploy: build payload bundle.yaml
+ @echo "Deploying $(APP_NAME)..."
+ @juju deploy ./bundle.yaml
+
+payload: $(ASSET)
+$(ASSET):
+ @echo "Building asset for $(BUILD_LABEL)..."
+ @$(MAKE) -C .. build-tarball
+
+clean:
+ @find . -name \*.pyc -delete
+ @find . -depth -name '__pycache__' -exec rm -rf '{}' \;
+ @rm -f bundle.yaml
+ @rm -f layer/*/codetree-collect-info.yaml
+ @rm -rf $(BUILDDIR) $(TMPDIR)
+
+lint: build
+ @echo "Linting charms..."
+ @set -e; for charm in $(CHARMS); do \
+ charm proof dist/builds/$$charm; \
+ done
+ @echo "Linting python sources..."
+ @flake8 layer $(CHARMS)
+
+publish: build lint | $(PUBLISHDIR)
+ @set -e; for charm in $(CHARMS); do \
+ if [ -d $(PUBLISHDIR)/$$charm ]; then \
+ git -C $(PUBLISHDIR)/$$charm pull; \
+ else \
+ git clone $(PUBLISH_REPO_PREFIX)$$charm \
+ $(PUBLISHDIR)/$$charm; \
+ fi; \
+ rsync -a -m --ignore-times --exclude .git --delete \
+ dist/builds/$$charm/ $(PUBLISHDIR)/$$charm/; \
+ git -C $(PUBLISHDIR)/$$charm add .; \
+ if [ "$$(git -C $(PUBLISHDIR)/$$charm status --porcelain || \
+ echo status failed)" ]; then \
+ git -C $(PUBLISHDIR)/$$charm \
+ -c user.name="$(GIT_USERNAME)" \
+ -c user.email="$(GIT_EMAIL)" \
+ commit -a \
+ -m "Build of $$charm from $(BUILD_LABEL)"; \
+ git -C $(PUBLISHDIR)/$$charm tag build/$(BUILD_LABEL); \
+ fi; \
+ git -C $(PUBLISHDIR)/$$charm push --tags origin master; \
+ done
+
+# Prepare a Jenkins-as-a-service container for charm building.
+setup-jenkaas:
+ sudo systemctl stop snapd.socket
+ sudo systemctl stop snapd
+ echo SNAPPY_STORE_NO_CDN=1 | sudo tee -a /etc/environment >/dev/null
+ echo SNAPPY_TESTING=1 | sudo tee -a /etc/environment >/dev/null
+ sudo systemctl start snapd.socket
+ sudo snap install --classic charm
+
+.PHONY: $(foreach charm,$(CHARMS),build-$(charm))
+.PHONY: all build clean deploy lint payload publish setup-jenkaas
diff --git a/charm/README.md b/charm/README.md
new file mode 100644
index 0000000..b171b80
--- /dev/null
+++ b/charm/README.md
@@ -0,0 +1,16 @@
+# Overview
+
+This charm provides a service for storing keys and signing messages.
+
+# Usage
+
+ $ juju add-model lp-signing
+ $ make deploy PIP_SOURCE_DIR=/path/to/dependencies
+
+... where `/path/to/dependencies` is the path to a clone of
+https://git.launchpad.net/~launchpad/lp-signing/+git/dependencies.
+
+# Contact Information
+
+Colin Watson <cjwatson@xxxxxxxxxxxxx>
+https://launchpad.net/~cjwatson
diff --git a/charm/bundle.yaml.in b/charm/bundle.yaml.in
new file mode 100644
index 0000000..b99a248
--- /dev/null
+++ b/charm/bundle.yaml.in
@@ -0,0 +1,25 @@
+series: bionic
+description: "lp-signing development bundle"
+applications:
+ haproxy:
+ charm: cs:haproxy
+ num_units: 1
+ options:
+ services: "[]"
+ ssl_cert: "%SSL_CERT%"
+ ssl_key: "%SSL_KEY%"
+ postgresql:
+ charm: cs:postgresql
+ num_units: 1
+ lp-signing:
+ charm: ./dist/builds/lp-signing
+ num_units: 1
+ options:
+ build_label: "%BUILD_LABEL%"
+ service_private_keys: '["%SERVICE_PRIVATE_KEY%"]'
+ resources:
+ lp-signing: "../build/%BUILD_LABEL%/lp-signing.tar.gz"
+relations:
+ - ["haproxy", "lp-signing"]
+ - ["postgresql:db", "lp-signing:db"]
+ - ["postgresql:db-admin", "lp-signing:db-admin"]
diff --git a/charm/dependencies.txt b/charm/dependencies.txt
new file mode 100644
index 0000000..693a966
--- /dev/null
+++ b/charm/dependencies.txt
@@ -0,0 +1,2 @@
+ols-layers git+ssh://git.launchpad.net/~cjwatson/ols-charm-deps/+git/ols-layers;revno=f48dedcf
+charm-wheels git+ssh://git.launchpad.net/~ubuntuone-hackers/ols-charm-deps/+git/wheels;revno=555749f1
diff --git a/charm/lp-signing/config.yaml b/charm/lp-signing/config.yaml
new file mode 100644
index 0000000..54d5f45
--- /dev/null
+++ b/charm/lp-signing/config.yaml
@@ -0,0 +1,8 @@
+options:
+ service_private_keys:
+ type: string
+ default: "[]"
+ description: >
+ A JSON-encoded list of base64-encoded private service keys. The first
+ key in the list is the preferred one; older keys may follow,
+ permitting rollover.
diff --git a/charm/lp-signing/layer.yaml b/charm/lp-signing/layer.yaml
new file mode 100644
index 0000000..66de87c
--- /dev/null
+++ b/charm/lp-signing/layer.yaml
@@ -0,0 +1,21 @@
+includes:
+ - layer:ols-wsgi
+ - layer:ols-pg
+options:
+ ols:
+ service_name: lp-signing
+ config_filename: service.conf
+ user: launchpad
+ tarball_payload: true
+ symlink_switch_payload: true
+ ols-http:
+ port: 8000
+ ols-wsgi:
+ app: lp_signing.webapp:app
+ ols-pg:
+ databases:
+ db:
+ name: lp_signing
+ roles: lp_signing
+ migrations: migrations
+repo: https://git.launchpad.net/lp-signing
diff --git a/charm/lp-signing/metadata.yaml b/charm/lp-signing/metadata.yaml
new file mode 100644
index 0000000..0a6bd38
--- /dev/null
+++ b/charm/lp-signing/metadata.yaml
@@ -0,0 +1,13 @@
+name: lp-signing
+display-name: lp-signing
+summary: Launchpad signing service
+maintainer: Colin Watson <cjwatson@xxxxxxxxxxxxx>
+description: A service for storing keys and signing messages.
+series:
+ - bionic
+subordinate: false
+resources:
+ lp-signing:
+ type: file
+ filename: lp-signing.tar.gz
+ description: Launchpad signing code
diff --git a/charm/lp-signing/reactive/lp-signing.py b/charm/lp-signing/reactive/lp-signing.py
new file mode 100644
index 0000000..ec512aa
--- /dev/null
+++ b/charm/lp-signing/reactive/lp-signing.py
@@ -0,0 +1,59 @@
+# Copyright 2018-2020 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from urllib.parse import (
+ urlparse,
+ urlunparse,
+ )
+
+from charmhelpers.core import (
+ hookenv,
+ templating,
+ )
+from charms.reactive import (
+ set_state,
+ when,
+ )
+from ols import (
+ base,
+ http,
+ postgres,
+ )
+
+
+def stormify_db_uri(uri):
+ """Storm requires postgres:// rather than postgresql://."""
+ parsed_uri = urlparse(uri)
+ scheme = parsed_uri.scheme
+ if scheme == 'postgresql':
+ scheme = 'postgres'
+ return urlunparse((scheme, *parsed_uri[1:]))
+
+
+@when('ols.configured', 'db.master.available')
+def configure(pgsql):
+ config = hookenv.config()
+ config_path = base.service_config_path()
+ svc_config = dict(config)
+
+ master, standbys = postgres.get_db_uris(pgsql)
+ svc_config['master_url'] = stormify_db_uri(master)
+ svc_config['standby_urls'] = [
+ stormify_db_uri(standby) for standby in standbys]
+
+ hookenv.log('Writing service config.')
+ templating.render(
+ 'service.conf.j2', config_path, svc_config,
+ owner='root', group=base.user(), perms=0o440)
+
+ set_state('service.configured')
+
+
+@when('service.configured', 'ols.wsgi.configured')
+def check_is_running():
+ if http.is_listening():
+ hookenv.status_set('active', 'Ready')
+ else:
+ hookenv.status_set('blocked', 'Service not running, check logs')
diff --git a/charm/lp-signing/templates/service.conf.j2 b/charm/lp-signing/templates/service.conf.j2
new file mode 100644
index 0000000..044fbca
--- /dev/null
+++ b/charm/lp-signing/templates/service.conf.j2
@@ -0,0 +1,9 @@
+[database]
+master_url: {{ master_url }}
+standby_urls: [{% if standby_urls %}"{{ standby_urls|join('", "') }}"{% endif %}]
+
+[auth]
+# The first key in this sequence is the preferred one. Older keys may
+# follow, permitting rollover.
+service_private_keys: {{ service_private_keys }}
+
diff --git a/charm/packages.txt b/charm/packages.txt
new file mode 100644
index 0000000..c33fce5
--- /dev/null
+++ b/charm/packages.txt
@@ -0,0 +1,3 @@
+flake8
+python-codetree
+squashfuse
diff --git a/ols-vms.conf b/ols-vms.conf
index d3189fa..9f6b8ac 100644
--- a/ols-vms.conf
+++ b/ols-vms.conf
@@ -3,7 +3,7 @@ vm.architecture = amd64
vm.release = bionic
apt.sources = ppa:launchpad/ppa
-vm.packages = @dependencies.txt, @dependencies-devel.txt
+vm.packages = @dependencies.txt, @dependencies-devel.txt, @charm/packages.txt
[lp-signing]
vm.class = lxd
diff --git a/requirements.txt b/requirements.txt
index 2c471e4..3f4d8c3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -14,12 +14,14 @@ jsonschema==2.6.0
lazr.enum==1.2
lazr.postgresql==0.0.4
MarkupSafe==1.1.0
+pip==19.0.2
psycopg2==2.7.7
PyNaCl==1.3.0
PyYAML==3.13
pytz==2019.3
raven==6.10.0
requests==2.22.0
+setuptools==42.0.2
six==1.13.0
sqlparse==0.2.4
statsd==3.3.0
@@ -27,3 +29,4 @@ storm==0.22
talisker==0.11.1
urllib3==1.25.7
Werkzeug==0.14.1
+wheel==0.33.1
