launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #30042
[Merge] txpkgupload:add-tarball-publishing-to-makefile into txpkgupload:master
Ines Almeida has proposed merging txpkgupload:add-tarball-publishing-to-makefile into txpkgupload:master.
Commit message:
Add commands to makefile to build and publish tarball
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~lazr-developers/txpkgupload/+git/txpkgupload/+merge/443805
These commands were based on similar ones found on 'lp:turnip', 'lp:lp-archive' and 'lp:lp-signing'.
These commands will be important to later run them in a Jenkins job, and for charming txpkgupload.
This new txpkgupload charm will be then used by other charms that are on the making (ppa publisher, ftp publisher...)
--
Your team Launchpad code reviewers is requested to review the proposed merge of txpkgupload:add-tarball-publishing-to-makefile into txpkgupload:master.
diff --git a/Makefile b/Makefile
index cf21735..9d72082 100644
--- a/Makefile
+++ b/Makefile
@@ -21,6 +21,18 @@ TWISTD = $(ENV)/bin/twistd
VERSION_INFO = version-info.py
+# Create archives in labelled directories (e.g.
+# <rev-id>/$(PROJECT_NAME).tar.gz)
+TARBALL_BUILD_LABEL ?= $(shell git rev-parse HEAD)
+TARBALL_FILE_NAME = txpkgupload.tar.gz
+TARBALL_BUILDS_DIR ?= build
+TARBALL_BUILD_DIR = $(TARBALL_BUILDS_DIR)/$(TARBALL_BUILD_LABEL)
+TARBALL_BUILD_PATH = $(TARBALL_BUILD_DIR)/$(TARBALL_FILE_NAME)
+
+SWIFT_CONTAINER_NAME ?= txpkgupload-builds
+SWIFT_OBJECT_PATH = $(SWIFT_CONTAINER_NAME)/$(TARBALL_BUILD_LABEL)/$\
+ $(TARBALL_FILE_NAME)
+
default: compile
@@ -83,3 +95,21 @@ clean: clean_pip
.PHONY: compile check clean clean_pip default dist
+
+
+build-tarball: $(DEPENDENCY_DIR)
+ @echo "Creating deployment tarball at $(TARBALL_BUILD_PATH)"
+ mkdir -p $(TARBALL_BUILD_DIR)
+ tar -czf $(TARBALL_BUILD_PATH) \
+ --exclude build \
+ --exclude dist \
+ --exclude charm \
+ --exclude env \
+ ./
+
+
+publish-tarball: build-tarball
+ [ ! -e ~/.config/swift/txpkgupload ] || . ~/.config/swift/txpkgupload; \
+ ./publish-to-swift --debug \
+ $(SWIFT_CONTAINER_NAME) $(SWIFT_OBJECT_PATH) \
+ $(TARBALL_BUILD_PATH)
diff --git a/dependencies-devel.txt b/dependencies-devel.txt
index 5f4f7c8..3644c7c 100644
--- a/dependencies-devel.txt
+++ b/dependencies-devel.txt
@@ -7,3 +7,4 @@ python3
python3-dev
tox
virtualenv
+python3-swiftclient
diff --git a/ols-vms.conf b/ols-vms.conf
index 9f75bec..bdc6348 100644
--- a/ols-vms.conf
+++ b/ols-vms.conf
@@ -4,6 +4,7 @@ vm.release = xenial
vm.packages = @dependencies-devel.txt
-[lp-signing]
+[txpkgupload]
vm.class = lxd
vm.update = True
+jenkaas.secrets = swift/txpkgupload:.config/swift/txpkgupload
diff --git a/publish-to-swift b/publish-to-swift
new file mode 100755
index 0000000..0388082
--- /dev/null
+++ b/publish-to-swift
@@ -0,0 +1,154 @@
+#! /usr/bin/python3
+
+"""Publish a built tarball to Swift for deployment."""
+
+import os
+import re
+import subprocess
+import sys
+from argparse import ArgumentParser
+
+
+def ensure_container_privs(container_name):
+ """Ensure that the container exists and is world-readable.
+
+ This allows us to give services suitable credentials for getting the
+ built code from a container.
+ """
+ subprocess.run(["swift", "post", container_name, "--read-acl", ".r:*"])
+
+
+def get_swift_storage_url():
+ # This is a bit cumbersome, but probably still easier than bothering
+ # with swiftclient.
+ auth = subprocess.run(
+ ["swift", "auth"],
+ stdout=subprocess.PIPE,
+ check=True,
+ text=True,
+ ).stdout.splitlines()
+ return [
+ line.split("=", 1)[1]
+ for line in auth
+ if line.startswith("export OS_STORAGE_URL=")
+ ][0]
+
+
+def publish_file_to_swift(
+ container_name, object_path, local_path, overwrite=True
+):
+ """Publish a file to a Swift container."""
+ storage_url = get_swift_storage_url()
+
+ already_published = False
+ # Some swift versions unhelpfully exit 0 regardless of whether the
+ # object exists.
+ try:
+ stats = subprocess.run(
+ ["swift", "stat", container_name, object_path],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.DEVNULL,
+ check=True,
+ text=True,
+ ).stdout
+ if re.search(
+ r"Object: %s$" % re.escape(object_path), stats, flags=re.M
+ ):
+ already_published = True
+ except subprocess.CalledProcessError:
+ pass
+
+ if already_published:
+ print(
+ "Object {} already published to {}.".format(
+ object_path, container_name
+ )
+ )
+ if not overwrite:
+ return
+
+ print(
+ "Publishing {} to {} as {}.".format(
+ local_path, container_name, object_path
+ )
+ )
+ try:
+ subprocess.run(
+ [
+ "swift",
+ "upload",
+ "--object-name",
+ object_path,
+ container_name,
+ local_path,
+ ]
+ )
+ except subprocess.CalledProcessError:
+ sys.exit(
+ "Failed to upload {} to {} as {}".format(
+ local_path, container_name, object_path
+ )
+ )
+
+ print(
+ "Published file: {}/{}/{}".format(
+ storage_url, container_name, object_path
+ )
+ )
+
+
+def main():
+ parser = ArgumentParser()
+ parser.add_argument("--debug", action="store_true", default=False)
+ parser.add_argument("container_name")
+ parser.add_argument("swift_object_path")
+ parser.add_argument("local_path")
+ args = parser.parse_args()
+
+ if args.debug:
+ # Print OpenStack-related environment variables for ease of
+ # debugging. Only OS_AUTH_TOKEN and OS_PASSWORD currently seem to
+ # be secret, but for safety we only show unredacted contents of
+ # variables specifically known to be safe. See "swift --os-help"
+ # for most of these.
+ safe_keys = {
+ "OS_AUTH_URL",
+ "OS_AUTH_VERSION",
+ "OS_CACERT",
+ "OS_CERT",
+ "OS_ENDPOINT_TYPE",
+ "OS_IDENTITY_API_VERSION",
+ "OS_INTERFACE",
+ "OS_KEY",
+ "OS_PROJECT_DOMAIN_ID",
+ "OS_PROJECT_DOMAIN_NAME",
+ "OS_PROJECT_ID",
+ "OS_PROJECT_NAME",
+ "OS_REGION_NAME",
+ "OS_SERVICE_TYPE",
+ "OS_STORAGE_URL",
+ "OS_TENANT_ID",
+ "OS_TENANT_NAME",
+ "OS_USERNAME",
+ "OS_USER_DOMAIN_ID",
+ "OS_USER_DOMAIN_NAME",
+ "OS_USER_ID",
+ }
+ for key, value in sorted(os.environ.items()):
+ if key.startswith("OS_"):
+ if key not in safe_keys:
+ value = "<redacted>"
+ print(f"{key}: {value}")
+
+ overwrite = "FORCE_REBUILD" in os.environ
+ ensure_container_privs(args.container_name)
+ publish_file_to_swift(
+ args.container_name,
+ args.swift_object_path,
+ args.local_path,
+ overwrite=overwrite,
+ )
+
+
+if __name__ == "__main__":
+ main()