launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #27810
[Merge] ~cjwatson/launchpad:build-tarball into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:build-tarball into launchpad:master with ~cjwatson/launchpad:version-info-outside-git as a prerequisite.
Commit message:
Support publishing deployment artifact to Swift
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/412692
This adds `build-tarball` and `publish-tarball` make targets, which will help us to write a Jenkins job that builds a deployment artifact and publishes it to Swift for use by future deployment machinery.
The new `ols-vms.conf` file is to assist integration with lp:ols-jenkaas, which is where we currently do publication to Swift for other projects.
Aside from the new build and publish scripts, I had to adjust the existing wheel-building target a bit to also build wheels for the packages in `requirements/setup.txt`.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:build-tarball into launchpad:master.
diff --git a/Makefile b/Makefile
index 8b00a2e..be3eb85 100644
--- a/Makefile
+++ b/Makefile
@@ -15,6 +15,8 @@ PY=$(WD)/bin/py
PYTHONPATH:=$(WD)/lib:${PYTHONPATH}
VERBOSITY=-vv
+DEPENDENCY_REPO ?= https://git.launchpad.net/lp-source-dependencies
+
# virtualenv and pip fail if setlocale fails, so force a valid locale.
PIP_ENV := LC_ALL=C.UTF-8
# Run with "make PIP_NO_INDEX=" if you want pip to find software
@@ -95,6 +97,21 @@ PIP_BIN = \
bin/watch_jsbuild \
bin/with-xvfb
+# Create archives in labelled directories (e.g.
+# <rev-id>/$(PROJECT_NAME).tar.gz)
+TARBALL_BUILD_LABEL ?= $(shell git rev-parse HEAD)
+TARBALL_FILE_NAME = launchpad.tar.gz
+TARBALL_BUILDS_DIR ?= dist
+TARBALL_BUILD_DIR = $(TARBALL_BUILDS_DIR)/$(TARBALL_BUILD_LABEL)
+TARBALL_BUILD_PATH = $(TARBALL_BUILD_DIR)/$(TARBALL_FILE_NAME)
+
+SWIFT_CONTAINER_NAME ?= launchpad-builds
+# This must match the object path used by fetch_payload in the ols charm
+# layer.
+SWIFT_OBJECT_PATH = \
+ launchpad-builds/$(TARBALL_BUILD_LABEL)/$(TARBALL_FILE_NAME)
+
+
# DO NOT ALTER : this should just build by default
.PHONY: default
default: inplace
@@ -173,6 +190,17 @@ inplace: build logs clean_logs codehosting-dir
.PHONY: build
build: compile apidoc jsbuild css_combine
+# Bootstrap download-cache and sourcecode. Useful for CI jobs that want to
+# set these up from scratch.
+.PHONY: bootstrap
+bootstrap:
+ if [ -d download-cache/.git ]; then \
+ git -C download-cache pull; \
+ else \
+ git clone --depth=1 $(DEPENDENCY_REPO) download-cache; \
+ fi
+ utilities/update-sourcecode
+
# LP_SOURCEDEPS_PATH should point to the sourcecode directory, but we
# want the parent directory where the download-cache and env directories
# are. We re-use the variable that is using for the rocketfuel-get script.
@@ -258,29 +286,48 @@ requirements/combined.txt: \
--include requirements/launchpad.txt \
>"$@"
-# This target is used by LOSAs to prepare a build to be pushed out to
-# destination machines. We only want wheels: they are the expensive bits,
-# and the other bits might run into problems like bug 575037. This target
-# runs pip, builds a wheelhouse with predictable paths that can be used even
-# if the build is pushed to a different path on the destination machines,
-# and then removes everything created except for the wheels.
-#
# It doesn't seem to be straightforward to build a wheelhouse of all our
# dependencies without also building a useless wheel of Launchpad itself;
# fortunately that doesn't take too long, and we just remove it afterwards.
-.PHONY: build_wheels
-build_wheels: $(PIP_BIN) requirements/combined.txt
+.PHONY: build_wheels_only
+build_wheels_only: $(PIP_BIN) requirements/combined.txt
$(RM) -r wheelhouse wheels
+ $(SHHH) $(PIP) wheel -w wheels -r requirements/setup.txt
$(SHHH) $(PIP) wheel \
-c requirements/setup.txt -c requirements/combined.txt \
-w wheels .
$(RM) wheels/lp-[0-9]*.whl
- $(MAKE) clean_pip
+
+# This target is used by deployment machinery to prepare a build to be
+# pushed out to destination machines. We only want wheels: they are the
+# expensive bits, and the other bits might run into problems like bug
+# 575037. This target runs pip, builds a wheelhouse with predictable paths
+# that can be used even if the build is pushed to a different path on the
+# destination machines, and then removes everything created except for the
+# wheels.
+.PHONY: build_wheels
+build_wheels: build_wheels_only
+ $(MAKE) clean_js clean_pip
# Compatibility
.PHONY: build_eggs
build_eggs: build_wheels
+# Build a tarball that can be unpacked and built on another machine,
+# including all the wheels we need. This will eventually supersede
+# build_wheels.
+.PHONY: build-tarball
+build-tarball:
+ utilities/build-tarball $(TARBALL_BUILD_DIR)
+
+# Publish a buildable tarball to Swift.
+.PHONY: publish-tarball
+publish-tarball: build-tarball
+ [ ! -e ~/.config/swift/launchpad ] || . ~/.config/swift/launchpad; \
+ utilities/publish-to-swift --debug \
+ $(SWIFT_CONTAINER_NAME) $(SWIFT_OBJECT_PATH) \
+ $(TARBALL_BUILD_PATH)
+
# setuptools won't touch files that would have the same contents, but for
# Make's sake we need them to get fresh timestamps, so we touch them after
# building.
diff --git a/ols-vms.conf b/ols-vms.conf
new file mode 100644
index 0000000..cd26bce
--- /dev/null
+++ b/ols-vms.conf
@@ -0,0 +1,11 @@
+# Options defined here provide defaults for all sections
+vm.architecture = amd64
+vm.release = xenial
+
+apt.sources = ppa:launchpad/ppa
+vm.packages = launchpad-dependencies
+
+[launchpad]
+vm.class = lxd
+vm.update = True
+jenkaas.secrets = swift/launchpad:.config/swift/launchpad
diff --git a/utilities/build-tarball b/utilities/build-tarball
new file mode 100755
index 0000000..70acfdb
--- /dev/null
+++ b/utilities/build-tarball
@@ -0,0 +1,39 @@
+#! /bin/sh
+#
+# Copyright 2021 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+set -e
+
+if [ -z "$1" ]; then
+ echo "Usage: $0 OUTPUT-DIRECTORY" >&2
+ exit 2
+fi
+output_dir="$1"
+
+# Build wheels to include in the distributed tarball.
+make build_wheels_only
+
+# Ensure that we have an updated idea of this tree's version.
+scripts/update-version-info.sh
+
+echo "Creating deployment tarball in $output_dir"
+
+# Prepare a file list.
+mkdir -p "$output_dir"
+(
+ git ls-files | sed 's,^,./,'
+ # Most of download-cache is unnecessary since we include a wheelhouse
+ # instead, but JavaScript-enabled builds need yarn.
+ find ./download-cache/dist/ -name yarn-\*.tar.gz
+ # The subdirectories of sourcecode may be symlinks. Force tar to
+ # include the actual files instead.
+ find ./sourcecode/*/ \
+ \( -name .bzr -o -name __pycache__ \) -prune -o -type f -print
+ echo ./version-info.py
+ find ./wheels/ -name \*.whl -print
+) | sort >"$output_dir/.files"
+
+# Create the tarball.
+tar -czf "$output_dir/launchpad.tar.gz" --no-recursion \
+ --files-from "$output_dir/.files"
diff --git a/utilities/publish-to-swift b/utilities/publish-to-swift
new file mode 100755
index 0000000..84de4c1
--- /dev/null
+++ b/utilities/publish-to-swift
@@ -0,0 +1,160 @@
+#! /usr/bin/python3 -S
+#
+# Copyright 2021 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Publish a built tarball to Swift for deployment."""
+
+import _pythonpath # noqa: F401
+
+from argparse import ArgumentParser
+import os
+
+import iso8601
+import requests
+from swiftclient.service import (
+ get_conn,
+ process_options,
+ SwiftService,
+ SwiftUploadObject,
+ )
+from swiftclient.shell import add_default_args
+
+
+def ensure_container_privs(options, container_name):
+ """Ensure that the container exists and is world-readable.
+
+ This allows us to give services suitable credentials for getting the
+ built code from a container.
+ """
+ options = dict(options)
+ options["read_acl"] = ".r:*"
+ with SwiftService(options=options) as swift:
+ swift.post(container=container_name)
+
+
+def get_swift_storage_url(options):
+ """Return the storage URL under which Swift objects are published."""
+ return get_conn(options).get_auth()[0]
+
+
+def publish_file_to_swift(options, container_name, object_path, local_path,
+ overwrite=True):
+ """Publish a file to a Swift container."""
+ storage_url = get_swift_storage_url(options)
+
+ with SwiftService(options=options) as swift:
+ stat_results = swift.stat(
+ container=container_name, objects=[object_path])
+ if stat_results and next(stat_results)["success"]:
+ print("Object {} already published to {}.".format(
+ object_path, container_name))
+ if not overwrite:
+ return
+
+ print("Publishing {} to {} as {}.".format(
+ local_path, container_name, object_path))
+ for r in swift.upload(
+ container_name,
+ [SwiftUploadObject(local_path, object_name=object_path)]):
+ if not r["success"]:
+ raise r["error"]
+
+ print("Published file: {}/{}/{}".format(
+ storage_url, container_name, object_path))
+
+
+def prune_old_files_from_swift(options, container_name, object_dir):
+ """Prune files from Swift that we no longer need."""
+ response = requests.head("https://launchpad.net/";)
+ response.raise_for_status()
+ production_revision = response.headers["X-VCS-Revision"]
+
+ with SwiftService(options=options) as swift:
+ objs = {}
+ production_mtime = None
+ for stats in swift.list(
+ container=container_name,
+ options={"prefix": "{}/".format(object_dir)}):
+ if not stats["success"]:
+ raise stats["error"]
+ for item in stats["listing"]:
+ if item.get("subdir") is None:
+ mtime = iso8601.parse_date(item["last_modified"])
+ objs[item["name"]] = mtime
+ if item["name"].startswith(
+ "{}/{}/".format(object_dir, production_revision)):
+ production_mtime = mtime
+
+ if production_mtime is None:
+ print(
+ "No file in {} corresponding to production revision {}; "
+ "not pruning.".format(container_name, production_revision))
+ return
+
+ for object_name, mtime in sorted(objs.items()):
+ if mtime < production_mtime:
+ print("Pruning {} (older than production)".format(object_name))
+ for r in swift.delete(
+ container=container_name, objects=[object_name]):
+ if not r["success"]:
+ raise r["error"]
+
+
+def main():
+ parser = ArgumentParser()
+ parser.add_argument("container_name")
+ parser.add_argument("swift_object_path")
+ parser.add_argument("local_path")
+ add_default_args(parser)
+ args = parser.parse_args()
+
+ if args.debug:
+ # Print OpenStack-related environment variables for ease of
+ # debugging. Only OS_AUTH_TOKEN and OS_PASSWORD currently seem to
+ # be secret, but for safety we only show unredacted contents of
+ # variables specifically known to be safe. See "swift --os-help"
+ # for most of these.
+ safe_keys = {
+ "OS_AUTH_URL",
+ "OS_AUTH_VERSION",
+ "OS_CACERT",
+ "OS_CERT",
+ "OS_ENDPOINT_TYPE",
+ "OS_IDENTITY_API_VERSION",
+ "OS_INTERFACE",
+ "OS_KEY",
+ "OS_PROJECT_DOMAIN_ID",
+ "OS_PROJECT_DOMAIN_NAME",
+ "OS_PROJECT_ID",
+ "OS_PROJECT_NAME",
+ "OS_REGION_NAME",
+ "OS_SERVICE_TYPE",
+ "OS_STORAGE_URL",
+ "OS_TENANT_ID",
+ "OS_TENANT_NAME",
+ "OS_USERNAME",
+ "OS_USER_DOMAIN_ID",
+ "OS_USER_DOMAIN_NAME",
+ "OS_USER_ID",
+ }
+ for key, value in sorted(os.environ.items()):
+ if key.startswith("OS_"):
+ if key not in safe_keys:
+ value = "<redacted>"
+ print("{}: {}".format(key, value))
+
+ options = vars(args)
+ process_options(options)
+
+ overwrite = "FORCE_REBUILD" in os.environ
+ ensure_container_privs(options, args.container_name)
+ publish_file_to_swift(
+ options, args.container_name, args.swift_object_path, args.local_path,
+ overwrite=overwrite)
+ prune_old_files_from_swift(
+ options, args.container_name, args.swift_object_path.split("/")[0])
+
+
+if __name__ == "__main__":
+ main()
