canonical-ubuntu-qa team mailing list archive

[Tim Andersson <mp+463593@xxxxxxxxxxxxxxxxxx>]

Tim Andersson has proposed merging ~andersson123/autopkgtest-cloud:swift-cleanup into autopkgtest-cloud:master.

Commit message:
feat: web: add script to cleanup broken swift results

When running seed-new-release, and copying over results from the last
release to the devel release, we encountered a lot of errors in which
the testinfo.json file (something created by autopkgtest) wasn't
decipherable as json.

This commit introduces a script which iterates through testinfo.json
files in swift, and any that aren't valid json, or are corrupted in any
way, are replaced with a dummy dictionary like so:
```
{
 "message": "This file has been added manually and is not a result
    of any test"
}
```

This should reduce the error messages when running seed-new-release, and
also, make it a bit more clear for developers browsing results - having
a clear message like this instead of just an empty file is more
explicit.

Requested reviews:
  Canonical's Ubuntu QA (canonical-ubuntu-qa)

For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/463593

Introduces a script which replaces empty or broken testinfo.json files with a file with a message stating that the file was not populated by an autopkgtest run but was instead, populated "manually".

Should reduce number of error messages when running seed-new-release or download-all-results, and also make it clearer to developers when they come across a "faulty" testinfo.json file
-- 
Your team Canonical's Ubuntu QA is requested to review the proposed merge of ~andersson123/autopkgtest-cloud:swift-cleanup into autopkgtest-cloud:master.

diff --git a/charms/focal/autopkgtest-web/webcontrol/swift-cleanup b/charms/focal/autopkgtest-web/webcontrol/swift-cleanup
new file mode 100755
index 0000000..72fa681
--- /dev/null
+++ b/charms/focal/autopkgtest-web/webcontrol/swift-cleanup
@@ -0,0 +1,178 @@
+#!/usr/bin/python3
+import configparser
+import io
+import itertools
+import json
+import logging
+import os
+import sys
+import tarfile
+import tempfile
+
+import swiftclient
+from distro_info import UbuntuDistroInfo
+
+SWIFT_CREDS_FILE = "/home/ubuntu/public-swift-creds"
+
+
+def fake_up_testinfo_json(
+    object_contents, swift_conn, container_name, object_name
+):
+    object_tar_bytes = io.BytesIO(object_contents)
+    temp_dir = tempfile.TemporaryDirectory()
+    result_dir = temp_dir.name + "/result"
+    os.makedirs(result_dir)
+    autopkgtest_result_files = {}
+    with tarfile.open(None, "r", object_tar_bytes) as tar:
+        # amend testinfo.json
+        try:
+            testinfo = json.loads(
+                tar.extractfile("testinfo.json").read().decode()
+            )
+        except Exception as _:
+            testinfo = {
+                "message": "This file has been added manually and is not a result of any test"
+            }
+        existing_files = tar.getnames()
+        for f in existing_files:
+            autopkgtest_result_files[f] = (
+                tar.extractfile(f).read().decode().strip()
+            )
+        # replace testinfo.json
+        autopkgtest_result_files["testinfo.json"] = testinfo
+    # write the files to new directory for tar'ing
+    for file_name, values in autopkgtest_result_files.items():
+        with open(f"{result_dir}/{file_name}", "w") as f:
+            if file_name.endswith(".json"):
+                json.dump(values, f)
+            else:
+                f.write(values)
+    # tar up the directory
+    output_result_tar = temp_dir.name + "/result.tar"
+    with tarfile.open(output_result_tar, "w") as tar:
+        for fn in os.listdir(result_dir):
+            p = os.path.join(result_dir, fn)
+            tar.add(p, arcname=fn)
+    # now need to upload the result.tar to swift
+    logging.info("Deleting old object %s ...", object_name)
+    try:
+        swift_conn.delete_object(container_name, object_name)
+    except Exception as e:
+        logging.error(f"swift delete failed with {e}")
+        return
+    logging.info("Uploading new object %s...", output_result_tar)
+    with open(output_result_tar, "rb") as f:
+        swift_conn.put_object(
+            container_name,
+            object_name,
+            contents=f,
+            content_type=None,
+            headers=None,
+            content_length=os.path.getsize(output_result_tar),
+        )
+    logging.info("New object uploaded.")
+    temp_dir.cleanup()
+
+
+def is_testinfo_object_integrity_ok(object_contents, object_name):
+    tar_bytes = io.BytesIO(object_contents)
+    try:
+        with tarfile.open(None, "r", tar_bytes) as tar:
+            _ = json.loads(tar.extractfile("testinfo.json").read().decode())
+    except (KeyError, ValueError, tarfile.TarError) as e:
+        logging.info(
+            "%s is damaged, with the following error: %s\nwill fix...",
+            object_name,
+            e,
+        )
+        return False
+    return True
+
+
+def fix_testinfo_jsons_for_release(release, swift_conn):
+    """Download new results from a swift container"""
+    container_name = "autopkgtest-" + release
+    logging.info(f"Fetching objects in container: {container_name}")
+
+    try:
+        _, objects = swift_conn.get_container(
+            container_name, full_listing=True
+        )
+        logging.info("%i objects fetched...", len(objects))
+        num_objects = len(objects)
+        ctr = 0
+        increment = 5
+        progress = increment
+        for obj in objects:
+            ctr += 1
+            percent = (ctr / num_objects) * 100
+            if percent > progress or ctr == 1:
+                logging.info("%i percent" % round((ctr / num_objects) * 100))
+                progress += increment
+            if obj["name"].endswith("result.tar"):
+                _, object_contents = swift_conn.get_object(
+                    container_name, obj["name"]
+                )
+                testinfo_integrity = is_testinfo_object_integrity_ok(
+                    object_contents, obj["name"]
+                )
+                if not testinfo_integrity:
+                    fake_up_testinfo_json(
+                        object_contents,
+                        swift_conn,
+                        container_name,
+                        obj["name"],
+                    )
+            sys.stdout.flush()
+    except swiftclient.ClientException as e:
+        logging.error(
+            "Something went wrong accessing container %s\nTraceback: %s"
+            % (container_name, str(e))
+        )
+        raise
+
+
+def get_swift_con():
+    swift_cfg = configparser.ConfigParser()
+    with open(SWIFT_CREDS_FILE) as fp:
+        swift_cfg.read_file(
+            itertools.chain(["[swift]"], fp), source=SWIFT_CREDS_FILE
+        )
+    swift_creds = {
+        "authurl": swift_cfg["swift"]["OS_AUTH_URL"],
+        "user": swift_cfg["swift"]["OS_USERNAME"],
+        "key": swift_cfg["swift"]["OS_PASSWORD"],
+        "os_options": {
+            "region_name": swift_cfg["swift"]["OS_REGION_NAME"],
+            "project_domain_name": swift_cfg["swift"][
+                "OS_PROJECT_DOMAIN_NAME"
+            ],
+            "project_name": swift_cfg["swift"]["OS_PROJECT_NAME"],
+            "user_domain_name": swift_cfg["swift"]["OS_USER_DOMAIN_NAME"],
+        },
+        "auth_version": 3,
+    }
+    return swiftclient.Connection(**swift_creds)
+
+
+def get_releases():
+    releases = list(
+        set(
+            UbuntuDistroInfo().supported() + UbuntuDistroInfo().supported_esm()
+        )
+    )
+    releases.sort(key=UbuntuDistroInfo().all.index, reverse=True)
+    return releases
+
+
+def main():
+    logging.basicConfig(level=logging.INFO)
+    logging.info("Setting up swift connection")
+    swift_conn = get_swift_con()
+    releases = get_releases()
+    for release in releases:
+        fix_testinfo_jsons_for_release(release, swift_conn)
+
+
+if __name__ == "__main__":
+    main()