launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #29811
[Merge] ~cjwatson/lp-mailman:build-tarball into lp-mailman:master
Colin Watson has proposed merging ~cjwatson/lp-mailman:build-tarball into lp-mailman:master.
Commit message:
Support publishing deployment artifact to Swift
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/lp-mailman/+git/lp-mailman/+merge/439660
This adds "build-tarball" and "publish-tarball" make targets, which will help us to write a Jenkins job that builds a deployment artifact and publishes it to Swift for use by future deployment machinery.
This is mostly lifted fairly directly from our other projects, such as lp-codeimport and lp-signing, with only minor modernization. It's part of the overall task of making it possible to deploy this project using Juju charms.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/lp-mailman:build-tarball into lp-mailman:master.
diff --git a/.gitignore b/.gitignore
index 63884ed..6e60468 100644
--- a/.gitignore
+++ b/.gitignore
@@ -34,4 +34,4 @@ run.gdb
callgrind.out.*
!logs/README.txt
/logs
-/wheelhouse
+/wheels
diff --git a/Makefile b/Makefile
index 9f358b7..97f125c 100644
--- a/Makefile
+++ b/Makefile
@@ -19,7 +19,7 @@ PIP_ENV := LC_ALL=C.UTF-8
# if it is going to be reviewed/merged/deployed.
PIP_NO_INDEX := 1
PIP_ENV += PIP_NO_INDEX=$(PIP_NO_INDEX)
-PIP_ENV += PIP_FIND_LINKS="file://$(WD)/wheelhouse/ file://$(DEPENDENCY_DIR)/"
+PIP_ENV += PIP_FIND_LINKS="file://$(WD)/wheels/ file://$(DEPENDENCY_DIR)/"
VIRTUALENV := $(PIP_ENV) virtualenv
PIP := PYTHONPATH= $(PIP_ENV) env/bin/pip --cache-dir=$(WD)/pip-cache/
@@ -47,6 +47,20 @@ PIP_BIN = \
bin/run \
bin/test
+# Create archives in labelled directories (e.g.
+# <rev-id>/$(PROJECT_NAME).tar.gz)
+TARBALL_BUILD_LABEL ?= $(shell git rev-parse HEAD)-xenial
+TARBALL_FILE_NAME = lp-mailman.tar.gz
+TARBALL_BUILD_DIR = dist/$(TARBALL_BUILD_LABEL)
+TARBALL_BUILD_PATH = $(TARBALL_BUILD_DIR)/$(TARBALL_FILE_NAME)
+
+SWIFT_CONTAINER_NAME ?= lp-mailman-builds
+# This must match the object path used by fetch_payload in the ols charm
+# layer.
+SWIFT_OBJECT_PATH = \
+ lp-mailman-builds/$(TARBALL_BUILD_LABEL)/$(TARBALL_FILE_NAME)
+
+
# DO NOT ALTER : this should just build by default
default: build logs clean_logs
@@ -92,23 +106,43 @@ else
$(MAKE) bootstrap
endif
-# This target is used by LOSAs to prepare a build to be pushed out to
-# destination machines. We only want wheels: they are the expensive bits,
-# and the other bits might run into problems like bug 575037. This target
-# runs pip, builds a wheelhouse with predictable paths that can be used even
-# if the build is pushed to a different path on the destination machines,
-# and then removes everything created except for the wheels.
-#
# It doesn't seem to be straightforward to build a wheelhouse of all our
# dependencies without also building a useless wheel of lp-mailman itself;
# fortunately that doesn't take too long, and we just remove it afterwards.
-build_wheels: $(PIP_BIN)
- $(RM) -r wheelhouse
+.PHONY: build_wheels_only
+build_wheels_only: $(PIP_BIN)
+ $(RM) -r wheelhouse wheels
+ $(SHHH) $(PIP) wheel -w wheels -r setup-requirements.txt
$(SHHH) $(PIP) wheel \
- -c setup-requirements.txt -c constraints.txt -w wheelhouse .
- $(RM) wheelhouse/lp_mailman-[0-9]*.whl
+ -c setup-requirements.txt -c constraints.txt -w wheels .
+ $(RM) wheels/lp_mailman-[0-9]*.whl
+
+# This target is used by deployment machinery to prepare a build to be
+# pushed out to destination machines. We only want wheels: they are the
+# expensive bits, and the other bits might run into problems like bug
+# 575037. This target runs pip, builds a wheelhouse with predictable paths
+# that can be used even if the build is pushed to a different path on the
+# destination machines, and then removes everything created except for the
+# wheels.
+.PHONY: build_wheels
+build_wheels: build_wheels_only
$(MAKE) clean_pip
+# Build a tarball that can be unpacked and built on another machine,
+# including all the wheels we need. This will eventually supersede
+# build_wheels.
+.PHONY: build-tarball
+build-tarball:
+ utilities/build-tarball $(TARBALL_BUILD_DIR)
+
+# Publish a buildable tarball to Swift.
+.PHONY: publish-tarball
+publish-tarball: build-tarball
+ [ ! -e ~/.config/swift/lp-mailman ] || . ~/.config/swift/lp-mailman; \
+ utilities/publish-to-swift --debug \
+ $(SWIFT_CONTAINER_NAME) $(SWIFT_OBJECT_PATH) \
+ $(TARBALL_BUILD_PATH)
+
# setuptools won't touch files that would have the same contents, but for
# Make's sake we need them to get fresh timestamps, so we touch them after
# building.
@@ -122,7 +156,7 @@ $(PY): $(DEPENDENCY_DIR) constraints.txt setup.py
$(VIRTUALENV) \
--python=$(PYTHON) --never-download \
--extra-search-dir=$(DEPENDENCY_DIR)/ \
- --extra-search-dir=$(WD)/wheelhouse/ \
+ --extra-search-dir=$(WD)/wheels/ \
env
ln -sfn env/bin bin
$(SHHH) $(PIP) install -r setup-requirements.txt
@@ -139,7 +173,7 @@ $(subst $(PY),,$(PIP_BIN)): $(PY)
compile: $(PY)
${SHHH} utilities/relocate-virtualenv env
${SHHH} LPCONFIG=${LPCONFIG} ${PY} -t buildmailman.py
- scripts/update-version-info.sh
+ [ ! -d .git ] || scripts/update-version-info.sh
run: build inplace stop
bin/run -r mailman -i $(LPCONFIG)
@@ -193,7 +227,7 @@ clean: clean_mailman clean_pip clean_logs
if test -f sourcecode/mailman/Makefile; then \
$(MAKE) -C sourcecode/mailman clean; \
fi
- $(RM) -r env wheelhouse
+ $(RM) -r env wheelhouse wheels
$(RM) -r build
$(RM) $(VERSION_INFO)
$(RM) -r /var/tmp/lperr \
@@ -216,5 +250,5 @@ tags: compile
$(CURDIR)/lib "$(SITE_PACKAGES)"
mv $@.new $@
-.PHONY: build_wheels check clean clean_logs clean_pip compile \
+.PHONY: check clean clean_logs clean_pip compile \
debug default realclean TAGS tags
diff --git a/ols-vms.conf b/ols-vms.conf
index 9e80725..f3cc9cb 100644
--- a/ols-vms.conf
+++ b/ols-vms.conf
@@ -3,8 +3,9 @@ vm.architecture = amd64
vm.release = xenial
apt.sources = ppa:launchpad/ppa
-vm.packages = @system-dependencies.txt
+vm.packages = @system-dependencies.txt, python3-swiftclient
[lp-mailman]
vm.class = lxd
vm.update = True
+jenkaas.secrets = swift/lp-mailman:.config/swift/lp-mailman
diff --git a/utilities/build-tarball b/utilities/build-tarball
new file mode 100755
index 0000000..dc54558
--- /dev/null
+++ b/utilities/build-tarball
@@ -0,0 +1,36 @@
+#! /bin/sh
+#
+# Copyright 2021-2023 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+set -e
+
+if [ -z "$1" ]; then
+ echo "Usage: $0 OUTPUT-DIRECTORY" >&2
+ exit 2
+fi
+output_dir="$1"
+
+# Build wheels to include in the distributed tarball.
+# XXX cjwatson 2021-12-10: It's unfortunate that this script is called from
+# the Makefile and also has to call out to the Makefile, but this is
+# difficult to disentangle until we refactor our build system to use
+# something higher-level than pip.
+make build_wheels_only
+
+# Ensure that we have an updated idea of this tree's version.
+scripts/update-version-info.sh
+
+echo "Creating deployment tarball in $output_dir"
+
+# Prepare a file list.
+mkdir -p "$output_dir"
+(
+ git ls-files | sed 's,^,./,'
+ echo ./version-info.py
+ find ./wheels/ -name \*.whl -print
+) | sort >"$output_dir/.files"
+
+# Create the tarball.
+tar -czf "$output_dir/lp-mailman.tar.gz" --no-recursion \
+ --files-from "$output_dir/.files"
diff --git a/utilities/publish-to-swift b/utilities/publish-to-swift
new file mode 100755
index 0000000..0f3670d
--- /dev/null
+++ b/utilities/publish-to-swift
@@ -0,0 +1,157 @@
+#! /usr/bin/python3
+#
+# Copyright 2021 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Publish a built tarball to Swift for deployment."""
+
+from argparse import ArgumentParser
+import os
+import re
+import subprocess
+import sys
+
+
+def ensure_container_privs(container_name):
+ """Ensure that the container exists and is world-readable.
+
+ This allows us to give services suitable credentials for getting the
+ built code from a container.
+ """
+ subprocess.run(["swift", "post", container_name, "--read-acl", ".r:*"])
+
+
+def get_swift_storage_url():
+ # This is a bit cumbersome, but probably still easier than bothering
+ # with swiftclient.
+ auth = subprocess.run(
+ ["swift", "auth"],
+ stdout=subprocess.PIPE,
+ check=True,
+ universal_newlines=True,
+ ).stdout.splitlines()
+ return [
+ line.split("=", 1)[1]
+ for line in auth
+ if line.startswith("export OS_STORAGE_URL=")
+ ][0]
+
+
+def publish_file_to_swift(
+ container_name, object_path, local_path, overwrite=True
+):
+ """Publish a file to a Swift container."""
+ storage_url = get_swift_storage_url()
+
+ already_published = False
+ # Some swift versions unhelpfully exit 0 regardless of whether the
+ # object exists.
+ try:
+ stats = subprocess.run(
+ ["swift", "stat", container_name, object_path],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.DEVNULL,
+ check=True,
+ universal_newlines=True,
+ ).stdout
+ if re.search(
+ r"Object: %s$" % re.escape(object_path), stats, flags=re.M
+ ):
+ already_published = True
+ except subprocess.CalledProcessError:
+ pass
+
+ if already_published:
+ print(
+ "Object {} already published to {}.".format(
+ object_path, container_name
+ )
+ )
+ if not overwrite:
+ return
+
+ print(
+ "Publishing {} to {} as {}.".format(
+ local_path, container_name, object_path
+ )
+ )
+ try:
+ subprocess.run(
+ [
+ "swift",
+ "upload",
+ "--object-name",
+ object_path,
+ container_name,
+ local_path,
+ ]
+ )
+ except subprocess.CalledProcessError:
+ sys.exit(
+ "Failed to upload {} to {} as {}".format(
+ local_path, container_name, object_path
+ )
+ )
+
+ print(
+ "Published file: {}/{}/{}".format(
+ storage_url, container_name, object_path
+ )
+ )
+
+
+def main():
+ parser = ArgumentParser()
+ parser.add_argument("--debug", action="store_true", default=False)
+ parser.add_argument("container_name")
+ parser.add_argument("swift_object_path")
+ parser.add_argument("local_path")
+ args = parser.parse_args()
+
+ if args.debug:
+ # Print OpenStack-related environment variables for ease of
+ # debugging. Only OS_AUTH_TOKEN and OS_PASSWORD currently seem to
+ # be secret, but for safety we only show unredacted contents of
+ # variables specifically known to be safe. See "swift --os-help"
+ # for most of these.
+ safe_keys = {
+ "OS_AUTH_URL",
+ "OS_AUTH_VERSION",
+ "OS_CACERT",
+ "OS_CERT",
+ "OS_ENDPOINT_TYPE",
+ "OS_IDENTITY_API_VERSION",
+ "OS_INTERFACE",
+ "OS_KEY",
+ "OS_PROJECT_DOMAIN_ID",
+ "OS_PROJECT_DOMAIN_NAME",
+ "OS_PROJECT_ID",
+ "OS_PROJECT_NAME",
+ "OS_REGION_NAME",
+ "OS_SERVICE_TYPE",
+ "OS_STORAGE_URL",
+ "OS_TENANT_ID",
+ "OS_TENANT_NAME",
+ "OS_USERNAME",
+ "OS_USER_DOMAIN_ID",
+ "OS_USER_DOMAIN_NAME",
+ "OS_USER_ID",
+ }
+ for key, value in sorted(os.environ.items()):
+ if key.startswith("OS_"):
+ if key not in safe_keys:
+ value = "<redacted>"
+ print("{}: {}".format(key, value))
+
+ overwrite = "FORCE_REBUILD" in os.environ
+ ensure_container_privs(args.container_name)
+ publish_file_to_swift(
+ args.container_name,
+ args.swift_object_path,
+ args.local_path,
+ overwrite=overwrite,
+ )
+
+
+if __name__ == "__main__":
+ main()
