canonical-ubuntu-qa team mailing list archive

[Tim Andersson <mp+456888@xxxxxxxxxxxxxxxxxx>]

Tim Andersson has proposed merging ~andersson123/autopkgtest-cloud:preserve-swift-copy-script into autopkgtest-cloud:master.

Requested reviews:
  Canonical's Ubuntu QA (canonical-ubuntu-qa)

For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/456888
-- 
Your team Canonical's Ubuntu QA is requested to review the proposed merge of ~andersson123/autopkgtest-cloud:preserve-swift-copy-script into autopkgtest-cloud:master.

diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/copy-swift-results b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/copy-swift-results
new file mode 100755
index 0000000..f835901
--- /dev/null
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/copy-swift-results
@@ -0,0 +1,126 @@
+#!/usr/bin/python3
+import configparser
+import os
+
+import swiftclient
+
+creds_file = "./swift-copy.conf"
+
+# swift-copy should look something like this:
+# [old]
+# OS_REGION_NAME=
+# OS_INTERFACE=
+# OS_AUTH_URL=
+# OS_PROJECT_DOMAIN_NAME=
+# OS_USERNAME=
+# OS_USER_DOMAIN_NAME=
+# OS_PROJECT_NAME=
+# OS_PASSWORD=
+# OS_IDENTITY_API_VERSION=
+
+# [new]
+# OS_REGION_NAME=
+# OS_INTERFACE=
+# OS_AUTH_URL=
+# OS_PROJECT_DOMAIN_NAME=
+# OS_USERNAME=
+# OS_USER_DOMAIN_NAME=
+# OS_PROJECT_NAME=
+# OS_PASSWORD=
+# OS_IDENTITY_API_VERSION=
+# But with the values filled out of course.
+
+releases = "noble mantic jammy focal lunar"
+
+
+conf = configparser.ConfigParser()
+conf.read(creds_file)
+
+swift_creds = {
+    "old": {
+        "authurl": conf["old"]["OS_AUTH_URL"],
+        "user": conf["old"]["OS_USERNAME"],
+        "key": conf["old"]["OS_PASSWORD"],
+        "os_options": {
+            "region_name": conf["old"]["OS_REGION_NAME"],
+            "project_domain_name": conf["old"]["OS_PROJECT_DOMAIN_NAME"],
+            "user_domain_name": conf["old"]["OS_USER_DOMAIN_NAME"],
+            "project_name": conf["old"]["OS_PROJECT_NAME"],
+        },
+        "auth_version": "3.0",
+    },
+    "new": {
+        "authurl": conf["new"]["OS_AUTH_URL"],
+        "user": conf["new"]["OS_USERNAME"],
+        "key": conf["new"]["OS_PASSWORD"],
+        "os_options": {
+            "region_name": conf["new"]["OS_REGION_NAME"],
+            "project_domain_name": conf["new"]["OS_PROJECT_DOMAIN_NAME"],
+            "user_domain_name": conf["new"]["OS_USER_DOMAIN_NAME"],
+            "project_name": conf["new"]["OS_PROJECT_NAME"],
+        },
+        "auth_version": "3.0",
+    },
+}
+
+swift_conn = {
+    "old": swiftclient.Connection(**swift_creds["old"]),
+    "new": swiftclient.Connection(**swift_creds["new"]),
+}
+
+for release in releases.split(" "):
+    if not os.path.isdir("items/autopkgtest-" + release):
+        os.mkdir("items/autopkgtest-" + release)
+    print("Getting items for autopkgtest-" + release + " from old storage")
+    _, object_list_old = swift_conn["old"].get_container(
+        "autopkgtest-" + release, full_listing=True
+    )
+    print("Getting items for autopkgtest-" + release + " from new storage")
+    _, object_list_new = swift_conn["new"].get_container(
+        "autopkgtest-" + release, full_listing=True
+    )
+    for obj_old in object_list_old:
+        print("Checking object: %s" % obj_old["name"])
+        already_in_new = False
+        for obj_new in object_list_new:
+            if obj_old["name"] == obj_new["name"]:
+                already_in_new = True
+                print(
+                    "Item %s already exists in new storage" % obj_old["name"]
+                )
+                break
+        if not already_in_new:
+            while True:
+                try:
+                    print(
+                        "Item doesn't exist in new db, downloading from old db!"
+                    )
+                    _, contents = swift_conn["old"].get_object(
+                        "autopkgtest-" + release, obj_old["name"]
+                    )
+                    print(
+                        "Item %s downloaded, putting in new db..."
+                        % obj_old["name"]
+                    )
+                    x = swift_conn["new"].put_object(
+                        "autopkgtest-" + release,
+                        obj_old["name"],
+                        contents,
+                        content_type="text/plain; charset=UTF-8"
+                        if ".gz" in obj_old["name"]
+                        else None,
+                        headers={"Content-Encoding": "gzip"}
+                        if ".gz" in obj_old["name"]
+                        else None,
+                    )
+                    print(
+                        "Item %s now placed into new storage!"
+                        % obj_old["name"]
+                    )
+                    break
+                except Exception as _:
+                    print("Failed! Restarting swift_conn")
+                    swift_conn = {
+                        "old": swiftclient.Connection(**swift_creds["old"]),
+                        "new": swiftclient.Connection(**swift_creds["new"]),
+                    }