launchpad-reviewers team mailing list archive

[Colin Watson <cjwatson@xxxxxxxxxxxxx>]

Colin Watson has proposed merging ~cjwatson/launchpad-buildd:charm into launchpad-buildd:master.

Commit message:
Add a Juju charm which can be used to deploy local launchpad-buildd instances

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~cjwatson/launchpad-buildd/+git/launchpad-buildd/+merge/383062

This is essentially the same as https://code.launchpad.net/~cjwatson/launchpad-buildd/charm/+merge/312468, converted to git and rebased on master.
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad-buildd:charm into launchpad-buildd:master.

diff --git a/.gitignore b/.gitignore
index b47264e..a939cb6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,8 @@
 *.pyc
 .pybuild
 buildd-slave-example.conf
+charm/dist
+charm/tmp
 dist
 debian/debhelper-build-stamp
 debian/files
diff --git a/charm/Makefile b/charm/Makefile
new file mode 100644
index 0000000..029cffc
--- /dev/null
+++ b/charm/Makefile
@@ -0,0 +1,77 @@
+NAME ?= launchpad-buildd
+TOPDIR ?= $(CURDIR)/..
+BUILDDIR ?= $(CURDIR)/dist
+TMPDIR ?= $(CURDIR)/tmp
+
+CHARM_SERIES ?= xenial
+CHARM_SRC ?= $(CURDIR)
+JUJU_REPOSITORY = $(BUILDDIR)
+CHARMDIR = $(BUILDDIR)/$(CHARM_SERIES)/$(NAME)
+CHARMREPODIR = $(BUILDDIR)/build
+RESOURCES = $(BUILDDIR)/resources/$(NAME)
+CHARM = $(CHARMDIR)/.done
+LAYER_PATH = $(TMPDIR)/layer
+INTERFACE_PATH = $(TMPDIR)/interface
+EXTRA_CHARM_BUILD_ARGS ?=
+DEPLOY_ENV ?= devel
+CHARM_STORE_URL ?= cs:~launchpad/launchpad-buildd
+
+export INTERFACE_PATH
+export LAYER_PATH
+export JUJU_REPOSITORY
+
+
+build: $(CHARM)
+
+packages: $(RESOURCES)
+
+build-with-packages: build packages
+
+$(BUILDDIR):
+	mkdir -p $@
+
+$(CHARM): $(CHARM_SRC) | $(BUILDDIR)
+	rm -rf $(CHARMDIR)
+	charm build -o $(BUILDDIR) -s $(CHARM_SERIES) -n $(NAME) $(EXTRA_CHARM_BUILD_ARGS)
+	touch $(CHARMDIR)/dummy-launchpad-buildd.deb
+	touch $(CHARMDIR)/dummy-python-lpbuildd.deb
+	touch $@
+
+clean:
+	rm -rf $(BUILDDIR) $(TMPDIR)
+
+$(RESOURCES):
+	rm -rf $(TMPDIR)
+	mkdir -p $(TMPDIR)
+	rsync -a --exclude /charm $(TOPDIR)/ $(TMPDIR)/$(NAME)/
+	cd $(TMPDIR)/$(NAME)/ && debuild -b -uc -us
+	mkdir -p $(RESOURCES)
+	cp -a $(TMPDIR)/launchpad-buildd_*.deb \
+	      $(RESOURCES)/launchpad-buildd.deb
+	cp -a $(TMPDIR)/python-lpbuildd_*.deb $(RESOURCES)/python-lpbuildd.deb
+
+create-privileged-model:
+	juju add-model privileged localhost
+	lxc profile set juju-privileged security.privileged true
+
+deploy:
+	juju deploy $(CHARMDIR)
+
+deploy-with-packages:
+	juju deploy \
+		--resource=launchpad-buildd=$(RESOURCES)/launchpad-buildd.deb \
+		--resource=python-lpbuildd=$(RESOURCES)/python-lpbuildd.deb \
+		$(CHARMDIR)
+
+push:
+	charm push $(CHARMDIR) $(CHARM_STORE_URL)
+
+push-with-packages:
+	charm push \
+		--resource=launchpad-buildd=$(CHARMDIR)/dummy-launchpad-buildd.deb \
+		--resource=python-lpbuildd=$(CHARMDIR)/dummy-python-lpbuildd.deb \
+		$(CHARMDIR) $(CHARM_STORE_URL)
+
+.PHONY: build packages build-with-packages clean
+.PHONY: create-privileged-model deploy deploy-with-packages
+.PHONY: push push-with-packages
diff --git a/charm/README.md b/charm/README.md
new file mode 100644
index 0000000..32e9259
--- /dev/null
+++ b/charm/README.md
@@ -0,0 +1,59 @@
+# Overview
+
+This charm installs a Launchpad builder, which can build packages in
+response to requests from a Launchpad instance.  It is mainly intended for
+use by Launchpad developers testing changes to builder handling.
+
+# Setup
+
+Builders need to be able to unpack chroots, which involves being able to
+create device nodes.  Unprivileged LXD containers cannot do this.  If you
+want to use this with the LXD provider, you should therefore do this first:
+
+```
+make create-privileged-model
+```
+
+... or, if you need more control, some variation on this:
+
+```
+juju add-model privileged localhost
+lxc profile set juju-privileged security.privileged true
+```
+
+# Deployment
+
+You can either deploy the stock launchpad-buildd package from a PPA, or
+build your own.
+
+Installing from a PPA is the default; just deploy this charm.
+
+If you're building your own package, then you already have the
+launchpad-buildd code checked out.  In the `charm/` subdirectory, build the
+charm and packages together:
+
+```
+make build-with-packages
+```
+
+Then deploy the charm, attaching the packages as resources:
+
+```
+make deploy-with-packages
+```
+
+Either way, this should eventually give you a running builder.  Find out its
+host name (e.g. `juju-XXXXXX-0.lxd`) and [add it to your local Launchpad
+instance](https://launchpad.test/builders/+new) (e.g.
+`http://juju-XXXXXX-0.lxd:8221/`).
+
+# Notes
+
+This charm gives you a non-virtualized builder, since there is no reset from
+a base image between builds; you'll need to make sure that any archives or
+snaps with builds you intend to dispatch to this builder have the "Require
+virtualized builders" option disabled.
+
+The Launchpad development wiki has [instructions on setting up the rest of
+Launchpad](https://dev.launchpad.net/Soyuz/HowToUseSoyuzLocally).
+You can skip the parts about installing the builder.
diff --git a/charm/config.yaml b/charm/config.yaml
new file mode 100644
index 0000000..c3a9340
--- /dev/null
+++ b/charm/config.yaml
@@ -0,0 +1,3 @@
+options:
+    install_sources:
+        default: ppa:launchpad/buildd-staging
diff --git a/charm/layer.yaml b/charm/layer.yaml
new file mode 100644
index 0000000..4ec9f2e
--- /dev/null
+++ b/charm/layer.yaml
@@ -0,0 +1,13 @@
+repo: lp:launchpad-buildd
+includes:
+    - layer:basic
+    - layer:apt
+options:
+    apt:
+        packages:
+            - bzr-builder
+            - git-build-recipe
+            - quilt
+ignore:
+    - dist
+    - tmp
diff --git a/charm/metadata.yaml b/charm/metadata.yaml
new file mode 100644
index 0000000..feacad0
--- /dev/null
+++ b/charm/metadata.yaml
@@ -0,0 +1,21 @@
+name: launchpad-buildd
+summary: Launchpad builder
+description: |
+    A system that can build packages in response to requests from a
+    Launchpad instance.
+tags:
+    - application_development
+maintainer: Colin Watson <cjwatson@xxxxxxxxxxxxx>
+subordinate: false
+series:
+    - xenial
+resources:
+    launchpad-buildd:
+        type: file
+        filename: launchpad-buildd.deb
+        description: The main Launchpad builder package.
+    python-lpbuildd:
+        type: file
+        filename: python-lpbuildd.deb
+        description: Python libraries supporting the Launchpad builder.
+min-juju-version: 2.0.0
diff --git a/charm/reactive/launchpad-buildd.py b/charm/reactive/launchpad-buildd.py
new file mode 100644
index 0000000..6ef8746
--- /dev/null
+++ b/charm/reactive/launchpad-buildd.py
@@ -0,0 +1,96 @@
+# Copyright 2016 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from __future__ import print_function
+
+import os.path
+import re
+import shutil
+import subprocess
+
+from charmhelpers import fetch
+from charmhelpers.core import (
+    hookenv,
+    host,
+    )
+from charms.apt import status_set
+from charms.reactive import (
+    hook,
+    only_once,
+    remove_state,
+    set_state,
+    when,
+    when_not,
+    )
+
+
+@only_once
+def install():
+    with open("/etc/default/launchpad-buildd", "w") as default_file:
+        print("RUN_NETWORK_REQUESTS_AS_ROOT=yes", file=default_file)
+    set_state("launchpad-buildd.needs_install")
+
+
+@hook("upgrade-charm", "config-changed")
+def mark_needs_install():
+    set_state("launchpad-buildd.needs_install")
+
+
+@when_not("apt.needs_update")
+@when("launchpad-buildd.needs_install")
+def install_packages():
+    cache_dir = os.path.join(hookenv.charm_dir(), "cache")
+    host.mkdir(cache_dir, perms=0o755)
+    to_install = []
+    packages = ["launchpad-buildd", "python-lpbuildd"]
+    options = ["--option=Dpkg::Options::=--force-confold"]
+    resource_paths = [hookenv.resource_get(package) for package in packages]
+    if all(path and os.path.getsize(path) for path in resource_paths):
+        # Install from resources.
+        changed = False
+        for package, resource_path in zip(packages, resource_paths):
+            local_path = os.path.join(cache_dir, "{}.deb".format(package))
+            to_install.append((local_path, resource_path))
+            if host.file_hash(local_path) != host.file_hash(resource_path):
+                changed = True
+        if not changed:
+            return
+        options.append("--reinstall")
+    else:
+        # We don't have resource-provided packages, so just install from the
+        # PPA.
+        to_install.extend([(None, package) for package in packages])
+    new_paths = [new_path for _, new_path in to_install]
+    try:
+        status_set(None, "Installing {}".format(",".join(packages)))
+        fetch.apt_unhold(packages)
+        fetch.apt_install(new_paths, options=options)
+        fetch.apt_hold(packages)
+    except subprocess.CalledProcessError:
+        status_set(
+            "blocked",
+            "Unable to install packages {}".format(",".join(packages)))
+    else:
+        for local_path, resource_path in to_install:
+            if local_path is not None:
+                shutil.copy2(resource_path, local_path)
+        # ntp.buildd isn't likely to work outside of the Canonical
+        # datacentre, and LXD containers can't set the system time.  Let's
+        # just not worry about NTP.
+        config_path = "/etc/launchpad-buildd/default"
+        with open(config_path) as config_file:
+            config = config_file.read()
+        config = re.sub(r"^ntphost = .*", "ntphost = ", config, flags=re.M)
+        with open(config_path + ".new", "w") as new_config_file:
+            new_config_file.write(config)
+        os.rename(config_path + ".new", config_path)
+        remove_state("launchpad-buildd.needs_install")
+        set_state("launchpad-buildd.installed")
+
+
+@when("apt.installed.bzr-builder")
+@when("apt.installed.git-build-recipe")
+@when("apt.installed.quilt")
+@when("launchpad-buildd.installed")
+def mark_active():
+    status_set("active", "Builder running")
diff --git a/debian/changelog b/debian/changelog
index e549316..9fb1cb9 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,8 @@
 launchpad-buildd (190) UNRELEASED; urgency=medium
 
   * Switch to git; add Vcs-* fields.
+  * Add a Juju charm which can be used to deploy local launchpad-buildd
+    instances.
 
  -- Colin Watson <cjwatson@xxxxxxxxxx>  Tue, 28 Apr 2020 10:19:27 +0100