launchpad-reviewers team mailing list archive

[Colin Watson <cjwatson@xxxxxxxxxxxxx>]

Colin Watson has proposed merging ~cjwatson/turnip:publish-tarball into turnip:master.

Commit message:
Add a "make publish-tarball" target

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~cjwatson/turnip/+git/turnip/+merge/385657

This will help us to write a Jenkins job that builds a deployment artifact and publishes it to Swift for use by the deployment machinery.

This is mostly copied from lp-signing.
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/turnip:publish-tarball into turnip:master.

diff --git a/Makefile b/Makefile
index 9e6bb12..3a8dfd0 100644
--- a/Makefile
+++ b/Makefile
@@ -27,6 +27,11 @@ TARBALL_BUILDS_DIR ?= build
 TARBALL_BUILD_DIR = $(TARBALL_BUILDS_DIR)/$(TARBALL_BUILD_LABEL)
 TARBALL_BUILD_PATH = $(TARBALL_BUILD_DIR)/$(TARBALL_FILE_NAME)
 
+SWIFT_CONTAINER_NAME ?= turnip-builds
+# This must match the object path used by install_payload in the turnip-base
+# charm layer.
+SWIFT_OBJECT_PATH = turnip-builds/$(TARBALL_BUILD_LABEL)/$(TARBALL_FILE_NAME)
+
 build: $(ENV)
 
 bootstrap:
@@ -113,4 +118,10 @@ build-tarball:
 		--exclude env \
 		./
 
+publish-tarball: build-tarball
+	[ ! -e ~/.config/swift/turnip ] || . ~/.config/swift/turnip; \
+	./publish-to-swift --debug \
+		$(SWIFT_CONTAINER_NAME) $(SWIFT_OBJECT_PATH) \
+		$(TARBALL_BUILD_PATH)
+
 .PHONY: build check clean dist lint run-api run-pack test build-tarball
diff --git a/dependencies-devel.txt b/dependencies-devel.txt
new file mode 100644
index 0000000..e3bee0e
--- /dev/null
+++ b/dependencies-devel.txt
@@ -0,0 +1 @@
+python3-swiftclient
diff --git a/ols-vms.conf b/ols-vms.conf
index 8d1324f..d7f8d17 100644
--- a/ols-vms.conf
+++ b/ols-vms.conf
@@ -4,7 +4,7 @@ vm.release = bionic
 
 # pygit2
 apt.sources = ppa:launchpad/ppa
-vm.packages = @system-dependencies.txt, @charm/packages.txt
+vm.packages = @system-dependencies.txt, @dependencies-devel.txt, @charm/packages.txt
 
 [turnip]
 vm.class = lxd
diff --git a/publish-to-swift b/publish-to-swift
new file mode 100755
index 0000000..e5cfd59
--- /dev/null
+++ b/publish-to-swift
@@ -0,0 +1,123 @@
+#! /usr/bin/python3
+
+"""Publish a built tarball to Swift for deployment."""
+
+from argparse import ArgumentParser
+import os
+import re
+import subprocess
+import sys
+
+
+def ensure_container_privs(container_name):
+    """Ensure that the container exists and is world-readable.
+
+    This allows us to give services suitable credentials for getting the
+    built code from a container.
+    """
+    subprocess.run(["swift", "post", container_name, "--read-acl", ".r:*"])
+
+
+def get_swift_storage_url():
+    # This is a bit cumbersome, but probably still easier than bothering
+    # with swiftclient.
+    auth = subprocess.run(
+        ["swift", "auth"],
+        stdout=subprocess.PIPE, check=True,
+        universal_newlines=True).stdout.splitlines()
+    return [
+        line.split("=", 1)[1] for line in auth
+        if line.startswith("export OS_STORAGE_URL=")][0]
+
+
+def publish_file_to_swift(container_name, object_path, local_path,
+                          overwrite=True):
+    """Publish a file to a Swift container."""
+    storage_url = get_swift_storage_url()
+
+    already_published = False
+    # Some swift versions unhelpfully exit 0 regardless of whether the
+    # object exists.
+    try:
+        stats = subprocess.run(
+            ["swift", "stat", container_name, object_path],
+            stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=True,
+            universal_newlines=True).stdout
+        if re.search(
+                r"Object: %s$" % re.escape(object_path), stats, flags=re.M):
+            already_published = True
+    except subprocess.CalledProcessError:
+        pass
+
+    if already_published:
+        print("Object {} already published to {}.".format(
+            object_path, container_name))
+        if not overwrite:
+            return
+
+    print("Publishing {} to {} as {}.".format(
+        local_path, container_name, object_path))
+    try:
+        subprocess.run(
+            ["swift", "upload", "--object-name", object_path,
+             container_name, local_path])
+    except subprocess.CalledProcessError:
+        sys.exit("Failed to upload {} to {} as {}".format(
+            local_path, container_name, object_path))
+
+    print("Published file: {}/{}/{}".format(
+        storage_url, container_name, object_path))
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument("--debug", action="store_true", default=False)
+    parser.add_argument("container_name")
+    parser.add_argument("swift_object_path")
+    parser.add_argument("local_path")
+    args = parser.parse_args()
+
+    if args.debug:
+        # Print OpenStack-related environment variables for ease of
+        # debugging.  Only OS_AUTH_TOKEN and OS_PASSWORD currently seem to
+        # be secret, but for safety we only show unredacted contents of
+        # variables specifically known to be safe.  See "swift --os-help"
+        # for most of these.
+        safe_keys = {
+            "OS_AUTH_URL",
+            "OS_AUTH_VERSION",
+            "OS_CACERT",
+            "OS_CERT",
+            "OS_ENDPOINT_TYPE",
+            "OS_IDENTITY_API_VERSION",
+            "OS_INTERFACE",
+            "OS_KEY",
+            "OS_PROJECT_DOMAIN_ID",
+            "OS_PROJECT_DOMAIN_NAME",
+            "OS_PROJECT_ID",
+            "OS_PROJECT_NAME",
+            "OS_REGION_NAME",
+            "OS_SERVICE_TYPE",
+            "OS_STORAGE_URL",
+            "OS_TENANT_ID",
+            "OS_TENANT_NAME",
+            "OS_USERNAME",
+            "OS_USER_DOMAIN_ID",
+            "OS_USER_DOMAIN_NAME",
+            "OS_USER_ID",
+            }
+        for key, value in sorted(os.environ.items()):
+            if key.startswith("OS_"):
+                if key not in safe_keys:
+                    value = "<redacted>"
+                print("{}: {}".format(key, value))
+
+    overwrite = "FORCE_REBUILD" in os.environ
+    ensure_container_privs(args.container_name)
+    publish_file_to_swift(
+        args.container_name, args.swift_object_path, args.local_path,
+        overwrite=overwrite)
+
+
+if __name__ == "__main__":
+    main()