canonical-ubuntu-qa team mailing list archive

[Tim Andersson <mp+471073@xxxxxxxxxxxxxxxxxx>]

Tim Andersson has proposed merging ~andersson123/autopkgtest-cloud:cleanup-old-systemd-upstream-jobs into autopkgtest-cloud:master.

Requested reviews:
  Canonical's Ubuntu QA (canonical-ubuntu-qa)

For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/471073

Remove results from systemd-upstream containers that are older than 30 days.
-- 
Your team Canonical's Ubuntu QA is requested to review the proposed merge of ~andersson123/autopkgtest-cloud:cleanup-old-systemd-upstream-jobs into autopkgtest-cloud:master.

diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/cleanup-old-upstream-results b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/cleanup-old-upstream-results
new file mode 100755
index 0000000..74f7730
--- /dev/null
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/cleanup-old-upstream-results
@@ -0,0 +1,92 @@
+#!/usr/bin/python3
+
+import argparse
+import datetime
+import logging
+import os
+import re
+
+import swiftclient
+
+UPSTREAM_RESULTS_RETENTION_TIME = 30  # days
+SWIFT_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
+SWIFT_RETRIES = 5
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="Script for removing upstream results older than 30 days."
+    )
+    parser.add_argument(
+        "--debug", "-d", action="store_true", help="debug mode"
+    )
+    return parser.parse_args()
+
+
+def connect_swift():
+    return swiftclient.Connection(
+        authurl=os.environ["SWIFT_AUTH_URL"],
+        user=os.environ["SWIFT_USERNAME"],
+        key=os.environ["SWIFT_PASSWORD"],
+        tenant_name=os.environ["SWIFT_PROJECT_NAME"],
+        os_options={"region_name": os.environ["SWIFT_REGION"]},
+        auth_version="3.0",
+        retries=10,
+        starting_backoff=10,
+    )
+
+
+def main():
+    args = parse_args()
+    if args.debug:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.INFO)
+    swift_con = connect_swift()
+    _, container_list = swift_con.get_account()
+    current_time = datetime.datetime.now()
+    upstream_regex = re.compile("^autopkgtest-[a-z]*-upstream-", re.DOTALL)
+    for container in container_list:
+        if not upstream_regex.search(container["name"]):
+            continue
+        logging.info(f"Examining container {container['name']}")
+        _, objects = swift_con.get_container(
+            container["name"], full_listing=True
+        )
+        num_deleted = 0
+        for object in objects:
+            object_datetime = datetime.datetime.strptime(
+                object["last_modified"], SWIFT_TIME_FORMAT
+            )
+            if (current_time - object_datetime) > datetime.timedelta(
+                days=UPSTREAM_RESULTS_RETENTION_TIME
+            ):
+                logging.debug(
+                    (
+                        f"Deleting object {object['name']} as it's older "
+                        f"than {UPSTREAM_RESULTS_RETENTION_TIME} days"
+                    )
+                )
+                for retry in range(SWIFT_RETRIES):
+                    try:
+                        swift_con.delete_object(
+                            container["name"], object["name"]
+                        )
+                        num_deleted += 1
+                        break
+                    except swiftclient.ClientException as e:
+                        logging.info(
+                            (
+                                f"Deleting object failed with {e} - "
+                                f"retrying {SWIFT_RETRIES - retry} more times"
+                            )
+                        )
+                        swift_con = connect_swift()
+        logging.info(
+            f"Deleted {num_deleted} of {len(objects)} from container {container['name']}"
+        )
+    logging.info("Done cleaning up old upstream swift results")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/charms/focal/autopkgtest-cloud-worker/units/cleanup-old-upstream-results.service b/charms/focal/autopkgtest-cloud-worker/units/cleanup-old-upstream-results.service
new file mode 100644
index 0000000..44aea91
--- /dev/null
+++ b/charms/focal/autopkgtest-cloud-worker/units/cleanup-old-upstream-results.service
@@ -0,0 +1,9 @@
+[Unit]
+Description=Delete old upstream results from swift
+
+[Service]
+Type=oneshot
+User=ubuntu
+Group=ubuntu
+EnvironmentFile=/home/ubuntu/swift-password.cred
+ExecStart=/home/ubuntu/autopkgtest-cloud/tools/cleanup-old-upstream-results
diff --git a/charms/focal/autopkgtest-cloud-worker/units/cleanup-old-upstream-results.timer b/charms/focal/autopkgtest-cloud-worker/units/cleanup-old-upstream-results.timer
new file mode 100644
index 0000000..3c72c87
--- /dev/null
+++ b/charms/focal/autopkgtest-cloud-worker/units/cleanup-old-upstream-results.timer
@@ -0,0 +1,8 @@
+[Unit]
+Description=Delete old upstream results from swift (timer)
+
+[Timer]
+OnCalendar=daily
+
+[Install]
+WantedBy=autopkgtest.target