launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #26215
[Merge] ~cjwatson/lp-codeimport:build-tarball into lp-codeimport:master
Colin Watson has proposed merging ~cjwatson/lp-codeimport:build-tarball into lp-codeimport:master with ~cjwatson/lp-codeimport:rename-wheelhouse as a prerequisite.
Commit message:
Support publishing deployment artifact to Swift
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/lp-codeimport/+git/lp-codeimport/+merge/397501
This adds "build-tarball" and "publish-tarball" make targets, which will help us to write a Jenkins job that builds a deployment artifact and publishes it to Swift for use by future deployment machinery.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/lp-codeimport:build-tarball into lp-codeimport:master.
diff --git a/Makefile b/Makefile
index 9e788a0..ae16dc5 100644
--- a/Makefile
+++ b/Makefile
@@ -8,6 +8,9 @@ PY=$(WD)/bin/py
PYTHONPATH:=$(WD)/lib:${PYTHONPATH}
VERBOSITY=-vv
+DEPENDENCY_REPO ?= https://git.launchpad.net/~launchpad/lp-codeimport/+git/dependencies
+DEPENDENCY_DIR ?= $(WD)/dependencies
+
# virtualenv and pip fail if setlocale fails, so force a valid locale.
PIP_ENV := LC_ALL=C.UTF-8
# Run with "make PIP_NO_INDEX=" if you want pip to find software
@@ -17,7 +20,7 @@ PIP_ENV := LC_ALL=C.UTF-8
# reviewed/merged/deployed.
PIP_NO_INDEX := 1
PIP_ENV += PIP_NO_INDEX=$(PIP_NO_INDEX)
-PIP_ENV += PIP_FIND_LINKS="file://$(WD)/wheels/ file://$(WD)/dependencies/"
+PIP_ENV += PIP_FIND_LINKS="file://$(WD)/wheels/ file://$(DEPENDENCY_DIR)/"
VIRTUALENV := $(PIP_ENV) virtualenv
PIP := PYTHONPATH= $(PIP_ENV) env/bin/pip --cache-dir=$(WD)/pip-cache/
@@ -42,6 +45,21 @@ PIP_BIN = \
bin/retest \
bin/test
+# Create archives in labelled directories (e.g.
+# <rev-id>/$(PROJECT_NAME).tar.gz)
+TARBALL_BUILD_LABEL ?= $(shell git rev-parse HEAD)
+TARBALL_FILE_NAME = lp-codeimport.tar.gz
+TARBALL_BUILDS_DIR ?= build
+TARBALL_BUILD_DIR = $(TARBALL_BUILDS_DIR)/$(TARBALL_BUILD_LABEL)
+TARBALL_BUILD_PATH = $(TARBALL_BUILD_DIR)/$(TARBALL_FILE_NAME)
+
+SWIFT_CONTAINER_NAME ?= lp-codeimport-builds
+# This must match the object path used by fetch_payload in the ols charm
+# layer.
+SWIFT_OBJECT_PATH = \
+ lp-codeimport-builds/$(TARBALL_BUILD_LABEL)/$(TARBALL_FILE_NAME)
+
+
# DO NOT ALTER : this should just build by default
default: inplace
@@ -64,15 +82,20 @@ inplace: build logs clean_logs
build: compile
+# Create or update $(DEPENDENCY_DIR) if possible. If the directory exists
+# but isn't a git repository, then assume that we're doing a deployment and
+# have already been provided with a suitable dependencies directory.
bootstrap:
- if [ -d dependencies ]; then \
- git -C dependencies pull; \
+ if [ -d $(DEPENDENCY_DIR)/.git ]; then \
+ git -C $(DEPENDENCY_DIR) pull; \
+ elif [ -d $(DEPENDENCY_DIR) ]; then \
+ :; \
else \
- git clone https://git.launchpad.net/~launchpad/lp-codeimport/+git/dependencies; \
+ git clone $(DEPENDENCY_REPO) $(DEPENDENCY_DIR); \
fi
utilities/update-sourcecode
-dependencies: bootstrap
+$(DEPENDENCY_DIR): bootstrap
requirements/combined.txt: \
requirements/setup.txt \
@@ -97,10 +120,11 @@ requirements/combined.txt: \
# afterwards.
build_wheels: $(PIP_BIN) requirements/combined.txt
$(RM) -r wheelhouse wheels
+ $(SHHH) $(PIP) wheel -r requirements/setup.txt -w wheels
$(SHHH) $(PIP) wheel \
-c requirements/setup.txt -c requirements/combined.txt \
-w wheels .
- $(RM) wheels/lp-codeimport-[0-9]*.whl
+ $(RM) wheels/lp_codeimport-[0-9]*.whl
$(MAKE) clean_pip
# setuptools won't touch files that would have the same contents, but for
@@ -110,12 +134,12 @@ build_wheels: $(PIP_BIN) requirements/combined.txt
# If we listed every target on the left-hand side, a parallel make would try
# multiple copies of this rule to build them all. Instead, we nominally build
# just $(PY), and everything else is implicitly updated by that.
-$(PY): dependencies requirements/combined.txt setup.py
+$(PY): $(DEPENDENCY_DIR) requirements/combined.txt setup.py
rm -rf env
mkdir -p env
$(VIRTUALENV) \
--python=$(PYTHON) --never-download \
- --extra-search-dir=$(WD)/dependencies/ \
+ --extra-search-dir=$(DEPENDENCY_DIR)/ \
--extra-search-dir=$(WD)/wheels/ \
env
ln -sfn env/bin bin
@@ -138,13 +162,33 @@ compile: $(PY)
fi
$(PYTHON) utilities/link-system-packages.py \
"$(SITE_PACKAGES)" system-packages.txt
- scripts/update-version-info.sh
+ [ ! -d .git ] || scripts/update-version-info.sh
$(VERSION_INFO):
scripts/update-version-info.sh
support_files: $(VERSION_INFO)
+# XXX cjwatson 2020-01-20: limit to only interesting files
+build-tarball: build_wheels
+ @echo "Creating deployment tarball at $(TARBALL_BUILD_PATH)"
+ scripts/update-version-info.sh
+ mkdir -p $(TARBALL_BUILD_DIR)
+ tar -czf $(TARBALL_BUILD_PATH) \
+ --exclude .git \
+ --exclude .gitignore \
+ --exclude build \
+ --exclude env \
+ --exclude logs \
+ --exclude pip-cache \
+ ./
+
+publish-tarball: build-tarball
+ [ ! -e ~/.config/swift/lp-codeimport ] || . ~/.config/swift/lp-codeimport; \
+ ./publish-to-swift --debug \
+ $(SWIFT_CONTAINER_NAME) $(SWIFT_OBJECT_PATH) \
+ $(TARBALL_BUILD_PATH)
+
clean_pip:
$(RM) -r build
$(RM) -r bin
diff --git a/ols-vms.conf b/ols-vms.conf
index db032d7..46544ea 100644
--- a/ols-vms.conf
+++ b/ols-vms.conf
@@ -3,7 +3,7 @@ vm.architecture = amd64
vm.release = xenial
apt.sources = ppa:launchpad/ppa
-vm.packages = @system-dependencies.txt
+vm.packages = @system-dependencies.txt, python3-swiftclient
[lp-codeimport]
vm.class = lxd
diff --git a/publish-to-swift b/publish-to-swift
new file mode 100755
index 0000000..e5cfd59
--- /dev/null
+++ b/publish-to-swift
@@ -0,0 +1,123 @@
+#! /usr/bin/python3
+
+"""Publish a built tarball to Swift for deployment."""
+
+from argparse import ArgumentParser
+import os
+import re
+import subprocess
+import sys
+
+
+def ensure_container_privs(container_name):
+ """Ensure that the container exists and is world-readable.
+
+ This allows us to give services suitable credentials for getting the
+ built code from a container.
+ """
+ subprocess.run(["swift", "post", container_name, "--read-acl", ".r:*"])
+
+
+def get_swift_storage_url():
+ # This is a bit cumbersome, but probably still easier than bothering
+ # with swiftclient.
+ auth = subprocess.run(
+ ["swift", "auth"],
+ stdout=subprocess.PIPE, check=True,
+ universal_newlines=True).stdout.splitlines()
+ return [
+ line.split("=", 1)[1] for line in auth
+ if line.startswith("export OS_STORAGE_URL=")][0]
+
+
+def publish_file_to_swift(container_name, object_path, local_path,
+ overwrite=True):
+ """Publish a file to a Swift container."""
+ storage_url = get_swift_storage_url()
+
+ already_published = False
+ # Some swift versions unhelpfully exit 0 regardless of whether the
+ # object exists.
+ try:
+ stats = subprocess.run(
+ ["swift", "stat", container_name, object_path],
+ stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=True,
+ universal_newlines=True).stdout
+ if re.search(
+ r"Object: %s$" % re.escape(object_path), stats, flags=re.M):
+ already_published = True
+ except subprocess.CalledProcessError:
+ pass
+
+ if already_published:
+ print("Object {} already published to {}.".format(
+ object_path, container_name))
+ if not overwrite:
+ return
+
+ print("Publishing {} to {} as {}.".format(
+ local_path, container_name, object_path))
+ try:
+ subprocess.run(
+ ["swift", "upload", "--object-name", object_path,
+ container_name, local_path])
+ except subprocess.CalledProcessError:
+ sys.exit("Failed to upload {} to {} as {}".format(
+ local_path, container_name, object_path))
+
+ print("Published file: {}/{}/{}".format(
+ storage_url, container_name, object_path))
+
+
+def main():
+ parser = ArgumentParser()
+ parser.add_argument("--debug", action="store_true", default=False)
+ parser.add_argument("container_name")
+ parser.add_argument("swift_object_path")
+ parser.add_argument("local_path")
+ args = parser.parse_args()
+
+ if args.debug:
+ # Print OpenStack-related environment variables for ease of
+ # debugging. Only OS_AUTH_TOKEN and OS_PASSWORD currently seem to
+ # be secret, but for safety we only show unredacted contents of
+ # variables specifically known to be safe. See "swift --os-help"
+ # for most of these.
+ safe_keys = {
+ "OS_AUTH_URL",
+ "OS_AUTH_VERSION",
+ "OS_CACERT",
+ "OS_CERT",
+ "OS_ENDPOINT_TYPE",
+ "OS_IDENTITY_API_VERSION",
+ "OS_INTERFACE",
+ "OS_KEY",
+ "OS_PROJECT_DOMAIN_ID",
+ "OS_PROJECT_DOMAIN_NAME",
+ "OS_PROJECT_ID",
+ "OS_PROJECT_NAME",
+ "OS_REGION_NAME",
+ "OS_SERVICE_TYPE",
+ "OS_STORAGE_URL",
+ "OS_TENANT_ID",
+ "OS_TENANT_NAME",
+ "OS_USERNAME",
+ "OS_USER_DOMAIN_ID",
+ "OS_USER_DOMAIN_NAME",
+ "OS_USER_ID",
+ }
+ for key, value in sorted(os.environ.items()):
+ if key.startswith("OS_"):
+ if key not in safe_keys:
+ value = "<redacted>"
+ print("{}: {}".format(key, value))
+
+ overwrite = "FORCE_REBUILD" in os.environ
+ ensure_container_privs(args.container_name)
+ publish_file_to_swift(
+ args.container_name, args.swift_object_path, args.local_path,
+ overwrite=overwrite)
+
+
+if __name__ == "__main__":
+ main()
