launchpad-reviewers team mailing list archive

[Colin Watson <cjwatson@xxxxxxxxxxxxx>]

Colin Watson has proposed merging ~cjwatson/lp-signing:publish-tarball into lp-signing:master.

Commit message:
Add a "make publish-tarball" target

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~cjwatson/lp-signing/+git/lp-signing/+merge/381367

This will help us to write a Jenkins job that builds a deployment artifact and publishes it to Swift for use by the deployment machinery.
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/lp-signing:publish-tarball into lp-signing:master.

diff --git a/Makefile b/Makefile
index 17013a5..f7eb6cb 100644
--- a/Makefile
+++ b/Makefile
@@ -30,6 +30,12 @@ TARBALL_BUILDS_DIR ?= build
 TARBALL_BUILD_DIR = $(TARBALL_BUILDS_DIR)/$(TARBALL_BUILD_LABEL)
 TARBALL_BUILD_PATH = $(TARBALL_BUILD_DIR)/$(TARBALL_FILE_NAME)
 
+SWIFT_CONTAINER_NAME ?= lp-signing-builds
+# This must match the object path used by fetch_payload in the ols charm
+# layer.
+SWIFT_OBJECT_PATH = \
+	lp-signing-builds/$(TARBALL_BUILD_LABEL)/$(TARBALL_FILE_NAME)
+
 
 $(ENV)/prod: | $(DEPENDENCY_DIR)
 	virtualenv $(ENV) --python=python3
@@ -94,6 +100,11 @@ build-tarball:
 		--exclude tmp \
 		./
 
+publish-tarball: build-tarball
+	./publish-to-swift \
+		$(SWIFT_CONTAINER_NAME) $(SWIFT_OBJECT_PATH) \
+		$(TARBALL_BUILD_PATH)
+
 include Makefile.db
 
 .PHONY: dbshell update-dependencies test clean coverage run migrate reset-db setup-db start-db stop-db
diff --git a/dependencies-devel.txt b/dependencies-devel.txt
index 7919215..496e3b5 100644
--- a/dependencies-devel.txt
+++ b/dependencies-devel.txt
@@ -3,4 +3,5 @@ make
 postgresql-contrib-10
 postgresql-server-dev-10
 python3-pip
+python3-swiftclient
 virtualenv
diff --git a/publish-to-swift b/publish-to-swift
new file mode 100755
index 0000000..967ad90
--- /dev/null
+++ b/publish-to-swift
@@ -0,0 +1,87 @@
+#! /usr/bin/python3
+
+"""Publish a built tarball to Swift for deployment."""
+
+from argparse import ArgumentParser
+import os
+import re
+import subprocess
+import sys
+
+
+def ensure_container_privs(container_name):
+    """Ensure that the container exists and is world-readable.
+
+    This allows us to give services suitable credentials for getting the
+    built code from a container.
+    """
+    subprocess.run(["swift", "post", container_name, "--read-acl", ".r:*"])
+
+
+def get_swift_storage_url():
+    # This is a bit cumbersome, but probably still easier than bothering
+    # with swiftclient.
+    auth = subprocess.run(
+        ["swift", "auth"],
+        stdout=subprocess.PIPE, check=True,
+        universal_newlines=True).stdout.splitlines()
+    return [
+        line.split("=", 1)[1] for line in auth
+        if line.startswith("export OS_STORAGE_URL=")][0]
+
+
+def publish_file_to_swift(container_name, object_path, local_path,
+                          overwrite=True):
+    """Publish a file to a Swift container."""
+    storage_url = get_swift_storage_url()
+
+    already_published = False
+    # Some swift versions unhelpfully exit 0 regardless of whether the
+    # object exists.
+    try:
+        stats = subprocess.run(
+            ["swift", "stat", container_name, object_path],
+            stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=True,
+            universal_newlines=True).stdout
+        if re.search(
+                r"Object: %s$" % re.escape(object_path), stats, flags=re.M):
+            already_published = True
+    except subprocess.CalledProcessError:
+        pass
+
+    if already_published:
+        print("Object {} already published to {}.".format(
+            object_path, container_name))
+        if not overwrite:
+            return
+
+    print("Publishing {} to {} as {}.".format(
+        local_path, container_name, object_path))
+    try:
+        subprocess.run(
+            ["swift", "upload", "--object-name", object_path,
+             container_name, local_path])
+    except subprocess.CalledProcessError:
+        sys.exit("Failed to upload {} to {} as {}".format(
+            local_path, container_name, object_path))
+
+    print("Published file: {}/{}/{}".format(
+        storage_url, container_name, object_path))
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument("container_name")
+    parser.add_argument("swift_object_path")
+    parser.add_argument("local_path")
+    args = parser.parse_args()
+
+    overwrite = "FORCE_REBUILD" in os.environ
+    ensure_container_privs(args.container_name)
+    publish_file_to_swift(
+        args.container_name, args.swift_object_path, args.local_path,
+        overwrite=overwrite)
+
+
+if __name__ == "__main__":
+    main()