canonical-ubuntu-qa team mailing list archive

[Tim Andersson <mp+463593@xxxxxxxxxxxxxxxxxx>]

Tim Andersson has proposed merging ~andersson123/autopkgtest-cloud:swift-cleanup into autopkgtest-cloud:master.

Requested reviews:
  Canonical's Ubuntu QA (canonical-ubuntu-qa)

For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/463593
-- 
Your team Canonical's Ubuntu QA is requested to review the proposed merge of ~andersson123/autopkgtest-cloud:swift-cleanup into autopkgtest-cloud:master.

diff --git a/charms/focal/autopkgtest-web/webcontrol/swift-cleanup b/charms/focal/autopkgtest-web/webcontrol/swift-cleanup
new file mode 100755
index 0000000..3b5b5f0
--- /dev/null
+++ b/charms/focal/autopkgtest-web/webcontrol/swift-cleanup
@@ -0,0 +1,170 @@
+#!/usr/bin/python3
+
+
+import configparser
+import io
+import itertools
+import json
+import logging
+import os
+import sys
+import tarfile
+import tempfile
+
+import swiftclient
+from distro_info import UbuntuDistroInfo
+
+LOGGER = logging.getLogger(__name__)
+SWIFT_CREDS_FILE = "/home/ubuntu/public-swift-creds"
+
+config = None
+
+
+def fix_broken_testinfo_json(
+    object_contents, swift_conn, container_name, object_name
+):
+    tar_bytes = io.BytesIO(object_contents)
+    temp_dir = tempfile.TemporaryDirectory()
+    tmp_dir = temp_dir.name
+    result_dir = tmp_dir + "/result"
+    os.makedirs(result_dir, exist_ok=True)
+    files_dict = {}
+    with tarfile.open(None, "r", tar_bytes) as tar:
+        # amend testinfo.json
+        try:
+            testinfo = json.loads(
+                tar.extractfile("testinfo.json").read().decode()
+            )
+        except Exception as _:
+            testinfo = {}
+        files = tar.getnames()
+        for f in files:
+            files_dict[f] = tar.extractfile(f).read().decode().strip()
+        files_dict["testinfo.json"] = testinfo
+    # write the files to new directory for tar'ing
+    for file_name, values in files_dict.items():
+        with open(f"{result_dir}/{file_name}", "w") as f:
+            if file_name.endswith(".json"):
+                json.dump(values, f)
+            else:
+                f.write(values)
+    # tar up the directory
+    output_filename = tmp_dir + "/result.tar"
+    with tarfile.open(output_filename, "w") as tar:
+        for fn in os.listdir(result_dir):
+            p = os.path.join(result_dir, fn)
+            tar.add(p, arcname=fn)
+    # now need to upload the result.tar to swift
+    logging.info("Deleting old object...")
+    try:
+        swift_conn.delete_object(container_name, object_name)
+    except Exception as e:
+        logging.error(f"swift delete failed with {e}")
+        return
+    logging.info("Uploading new object...")
+    with open(output_filename, "rb") as f:
+        swift_conn.put_object(
+            container_name,
+            object_name,
+            contents=f,
+            content_type=None,
+            headers=None,
+            content_length=os.path.getsize(output_filename),
+        )
+    logging.info("New object uploaded.")
+    temp_dir.cleanup()
+
+
+def check_object_integrity(object_contents, object_name):
+    tar_bytes = io.BytesIO(object_contents)
+    try:
+        with tarfile.open(None, "r", tar_bytes) as tar:
+            _ = json.loads(tar.extractfile("testinfo.json").read().decode())
+    except (KeyError, ValueError, tarfile.TarError) as e:
+        # here, I should amend testinfo.json and reupload
+        logging.info("%s is damaged, ignoring: %s" % (object_name, str(e)))
+        return False
+    return True
+
+
+def fetch_container(release, swift_conn):
+    """Download new results from a swift container"""
+    container_name = "autopkgtest-" + release
+    logging.info(f"Fetching objects in container: {container_name}")
+
+    try:
+        _, objects = swift_conn.get_container(
+            container_name, full_listing=True
+        )
+        logging.info("Objects fetched...")
+        num_objects = len(objects)
+        ctr = 0
+        increment = 10
+        for obj in objects:
+            ctr += 1
+            percent = (ctr / num_objects) * 100
+            if percent > increment or ctr == 1:
+                logging.info("%f percent" % ((ctr / num_objects) * 100))
+                increment += 10
+            if obj["name"].endswith("result.tar"):
+                _, contents = swift_conn.get_object(
+                    container_name, obj["name"]
+                )
+                integrity = check_object_integrity(contents, obj["name"])
+                if not integrity:
+                    fix_broken_testinfo_json(
+                        contents, swift_conn, container_name, obj["name"]
+                    )
+            sys.stdout.flush()
+    except swiftclient.ClientException as e:
+        LOGGER.error(
+            "Something went wrong accessing container %s\nTraceback: %s"
+            % (container_name, str(e))
+        )
+        raise
+
+
+def get_swift_con():
+    swift_cfg = configparser.ConfigParser()
+    with open(SWIFT_CREDS_FILE) as fp:
+        swift_cfg.read_file(
+            itertools.chain(["[swift]"], fp), source=SWIFT_CREDS_FILE
+        )
+    swift_creds = {
+        "authurl": swift_cfg["swift"]["OS_AUTH_URL"],
+        "user": swift_cfg["swift"]["OS_USERNAME"],
+        "key": swift_cfg["swift"]["OS_PASSWORD"],
+        "os_options": {
+            "region_name": swift_cfg["swift"]["OS_REGION_NAME"],
+            "project_domain_name": swift_cfg["swift"][
+                "OS_PROJECT_DOMAIN_NAME"
+            ],
+            "project_name": swift_cfg["swift"]["OS_PROJECT_NAME"],
+            "user_domain_name": swift_cfg["swift"]["OS_USER_DOMAIN_NAME"],
+        },
+        "auth_version": 3,
+    }
+    return swiftclient.Connection(**swift_creds)
+
+
+def get_releases():
+    releases = list(
+        set(
+            UbuntuDistroInfo().supported() + UbuntuDistroInfo().supported_esm()
+        )
+    )
+    releases.sort(key=UbuntuDistroInfo().all.index, reverse=True)
+    return releases
+
+
+def main():
+    logging.basicConfig(level=logging.INFO)
+    logging.info("Setting up swift connection")
+    swift_conn = get_swift_con()
+    releases = get_releases()
+    for release in releases:
+        fetch_container(release, swift_conn)
+
+
+if __name__ == "__main__":
+    main()