canonical-ubuntu-qa team mailing list archive

[Tim Andersson <mp+463593@xxxxxxxxxxxxxxxxxx>]

Tim Andersson has proposed merging ~andersson123/autopkgtest-cloud:swift-cleanup into autopkgtest-cloud:master.

Requested reviews:
  Canonical's Ubuntu QA (canonical-ubuntu-qa)

For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/463593
-- 
Your team Canonical's Ubuntu QA is requested to review the proposed merge of ~andersson123/autopkgtest-cloud:swift-cleanup into autopkgtest-cloud:master.

diff --git a/charms/focal/autopkgtest-web/webcontrol/swift-cleanup b/charms/focal/autopkgtest-web/webcontrol/swift-cleanup
new file mode 100644
index 0000000..ff7a43e
--- /dev/null
+++ b/charms/focal/autopkgtest-web/webcontrol/swift-cleanup
@@ -0,0 +1,126 @@
+#!/usr/bin/python3
+
+
+import configparser
+import datetime
+import io
+import itertools
+import json
+import logging
+import os
+import sqlite3
+import sys
+import tarfile
+import time
+import urllib.parse
+
+import amqplib.client_0_8 as amqp
+import swiftclient
+from distro_info import UbuntuDistroInfo
+from helpers.utils import SqliteWriterConfig, get_autopkgtest_cloud_conf
+
+
+LOGGER = logging.getLogger(__name__)
+SWIFT_CREDS_FILE = "/home/ubuntu/public-swift-creds"
+
+config = None
+
+
+def check_object_integrity(object_contents, object_name):
+    (_, _, _, src, _, _) = object_name.split("/")
+    tar_bytes = io.BytesIO(object_contents)
+    try:
+        with tarfile.open(None, "r", tar_bytes) as tar:
+            exitcode = int(tar.extractfile("exitcode").read().strip())
+            try:
+                srcver = (
+                    tar.extractfile("testpkg-version").read().decode().strip()
+                )
+            except KeyError:
+                # not found
+                if exitcode in (4, 12, 20):
+                    # repair it
+                    srcver = "%s unknown" % (src)
+                else:
+                    raise
+            (_, _) = srcver.split()
+            testinfo = json.loads(
+                tar.extractfile("testinfo.json").read().decode()
+            )
+            _ = testinfo.get("uuid", "")
+            _ = int(tar.extractfile("duration").read().strip())
+            # KeyError means the file is not there, i.e. there isn't a human
+            # requester
+            try:
+                _ = (
+                    tar.extractfile("requester").read().decode().strip()
+                )
+            except KeyError:
+                _ = ""
+    except (KeyError, ValueError, tarfile.TarError) as e:
+        LOGGER.debug("%s is damaged, ignoring: %s" % (object_name, str(e)))
+        return False
+    return True
+
+
+def fetch_container(release, swift_conn):
+    """Download new results from a swift container"""
+    container_name = "autopkgtest-" + release
+
+    try:
+        _, objects = swift_conn.get_container(container_name, full_listing=True)
+        for obj in objects:
+            _, contents = swift_conn.get_object(obj["name"])
+            integrity = check_object_integrity(contents, obj["name"])
+            if not integrity:
+                print(obj["name"])
+    except swiftclient.ClientException as e:
+        LOGGER.error(
+            "Something went wrong accessing container %s\nTraceback: %s"
+            % (container_name, str(e))
+        )
+        raise
+
+
+def get_swift_con():
+    swift_cfg = configparser.ConfigParser()
+    with open(SWIFT_CREDS_FILE) as fp:
+        swift_cfg.read_file(
+            itertools.chain(["[swift]"], fp), source=SWIFT_CREDS_FILE
+        )
+    swift_creds = {
+        "authurl": swift_cfg["swift"]["OS_AUTH_URL"],
+        "user": swift_cfg["swift"]["OS_USERNAME"],
+        "key": swift_cfg["swift"]["OS_PASSWORD"],
+        "os_options": {
+            "region_name": swift_cfg["swift"]["OS_REGION_NAME"],
+            "project_domain_name": swift_cfg["swift"][
+                "OS_PROJECT_DOMAIN_NAME"
+            ],
+            "project_name": swift_cfg["swift"]["OS_PROJECT_NAME"],
+            "user_domain_name": swift_cfg["swift"]["OS_USER_DOMAIN_NAME"],
+        },
+        "auth_version": 3,
+    }
+    return swiftclient.Connection(**swift_creds)
+
+
+def get_releases():
+    releases = list(
+        set(
+            UbuntuDistroInfo().supported() + UbuntuDistroInfo().supported_esm()
+        )
+    )
+    releases.sort(key=UbuntuDistroInfo().all.index, reverse=True)
+    return releases
+
+
+def main():
+    swift_conn = get_swift_con()
+    releases = get_releases()
+    for release in releases:
+        fetch_container(release, swift_conn)
+
+
+if __name__ == "__main__":
+    main()