canonical-ubuntu-qa team mailing list archive

[Tim Andersson <mp+456538@xxxxxxxxxxxxxxxxxx>]

Tim Andersson has proposed merging ~andersson123/autopkgtest-cloud:backup-worker-logs into autopkgtest-cloud:master.

Requested reviews:
  Canonical's Ubuntu QA (canonical-ubuntu-qa)

For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/456538
-- 
Your team Canonical's Ubuntu QA is requested to review the proposed merge of ~andersson123/autopkgtest-cloud:backup-worker-logs into autopkgtest-cloud:master.

diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/store-worker-logs b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/store-worker-logs
new file mode 100755
index 0000000..de803ea
--- /dev/null
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/store-worker-logs
@@ -0,0 +1,137 @@
+#!/usr/bin/python3
+
+
+import argparse
+import datetime
+import logging
+import os
+import subprocess
+import tempfile
+
+import swiftclient
+
+JOURNAL_CMD = ["journalctl", "--since", "6 hours ago", "--no-pager"]
+CONTAINER_NAME = "worker-logs"
+LOGFILE_LIFESPAN = 2678400 * 6  # 3 months
+
+logging.basicConfig(level="DEBUG" if "DEBUG" in os.environ else "INFO")
+
+# must have openstack-creds as environment file - similar to worker
+# EnvironmentFile=/home/ubuntu/swift-password.cred as env file
+# twice a day
+# journalctl --since "6 hours ago" > pipe to file with correct timestamp etc
+# write to swift container - worker-logs
+# use object file path to make it easy to find, by date
+# e.g.
+# 2023/11/24/worker-mm-HH-YYYY-MM-DD.log.gz
+# make sure to zip up the file and set the content encoding
+# upload object to container
+# done
+# maybe add a secondary functionality which cleans up old logs?
+
+# container called what? "worker-logs"?
+
+logging.info("Beginning store-worker-logs...")
+
+
+def acquire_journal(jrnl_opts):
+    this_cmd = JOURNAL_CMD
+    for i in jrnl_opts:
+        this_cmd.append(i)
+    return (
+        subprocess.Popen(this_cmd, stdout=subprocess.PIPE)
+        .stdout.read()
+        .decode("utf-8")
+    )
+
+
+logging.debug("Loading swift environmental variables...")
+
+swift_creds = {
+    "authurl": os.environ["SWIFT_AUTH_URL"],
+    "user": os.environ["SWIFT_USERNAME"],
+    "key": os.environ["SWIFT_PASSWORD"],
+    "os_options": {
+        "region_name": os.environ["SWIFT_REGION"],
+        "project_domain_name": os.environ["SWIFT_PROJECT_DOMAIN_NAME"],
+        "project_name": os.environ["SWIFT_PROJECT_NAME"],
+        "user_domain_name": os.environ["SWIFT_USER_DOMAIN_NAME"],
+    },
+    "auth_version": "3.0",
+}
+
+swift_conn = swiftclient.Connection(**swift_creds)
+logging.debug("swift connection set up!")
+parser = argparse.ArgumentParser()
+parser.add_argument("-d", "--dry-run", action="store_true")
+args = parser.parse_args()
+
+if not args.dry_run:
+    # check if container exists, if not create it
+    try:
+        logging.debug("Attempting to create container %s" % CONTAINER_NAME)
+        swift_conn.put_container(CONTAINER_NAME)
+    except Exception as e:  # XXX: amend me
+        logging.debug(
+            "Something went wrong with the container! Traceback: %s" % e
+        )
+
+rc_files = os.listdir("/home/ubuntu/cloudrcs/")
+res = {}
+temp_dir = tempfile.TemporaryDirectory()
+
+logging.info("Beginning to acquire logfiles...")
+for file in rc_files:
+    loc_arch = file.replace(".rc", "")
+    logging.info("Acquiring logfile for %s" % loc_arch)
+    jrnl_args = "-u autopkgtest@" + loc_arch + "*"
+    journal = acquire_journal(jrnl_args.split(" "))
+    fp = os.path.join(temp_dir.name, loc_arch + "_log")
+    with open(fp, "w") as logfile:
+        logfile.write(journal)
+    res[loc_arch] = fp
+
+curr_time = datetime.datetime.now()
+for loc_arch, log_fp in res.items():
+    object_path = os.path.join(
+        curr_time.strftime("%Y/%m/%d"),
+        loc_arch.replace("-", "/"),
+        curr_time.strftime("%H-%M"),
+        "log.gz",
+    )
+
+    if not args.dry_run:
+        with open(res[loc_arch]) as fd:
+            logging.info("Putting %s into swift storage!" % res[loc_arch])
+            content_type = "text/plain; charset=UTF-8"
+            headers = {"Content-Encoding": "gzip"}
+            swift_conn.put_object(
+                container=CONTAINER_NAME,
+                obj=object_path,
+                contents=fd,
+                content_type=content_type,
+                headers=headers,
+                content_length=os.path.getsize(res[loc_arch]),
+            )
+            logging.debug(
+                "%s placed into swift storage successfully as %s"
+                % (res[loc_arch], object_path)
+            )
+
+logging.info("worker logs successfully placed into swift storage.")
+
+temp_dir.cleanup()
+
+headers, objects = swift_conn.get_container(CONTAINER_NAME)
+now = datetime.datetime.now()
+for obj in objects:
+    object_age = datetime.datetime.strptime(
+        obj["last_modified"].split(".")[0], "%Y-%m-%dT%H:%M:%S"
+    )
+    if abs(now - object_age) > datetime.timedelta(days=180):
+        logging.info(
+            "Object %s is older than 180 days, deleting." % obj["name"]
+        )
+        swift_conn.delete_object(CONTAINER_NAME, obj["name"])
+
+swift_conn.close()
diff --git a/charms/focal/autopkgtest-cloud-worker/units/store-worker-logs.service b/charms/focal/autopkgtest-cloud-worker/units/store-worker-logs.service
new file mode 100644
index 0000000..ffa1f41
--- /dev/null
+++ b/charms/focal/autopkgtest-cloud-worker/units/store-worker-logs.service
@@ -0,0 +1,10 @@
+[Unit]
+Description=Store worker logs
+
+[Service]
+Type=oneshot
+User=ubuntu
+Group=ubuntu
+EnvironmentFile=/home/ubuntu/swift-password.cred
+ExecStart=/home/ubuntu/autopkgtest-cloud/tools/store-worker-logs
+TimeoutStartSec=10m
diff --git a/charms/focal/autopkgtest-cloud-worker/units/store-worker-logs.timer b/charms/focal/autopkgtest-cloud-worker/units/store-worker-logs.timer
new file mode 100644
index 0000000..71e5749
--- /dev/null
+++ b/charms/focal/autopkgtest-cloud-worker/units/store-worker-logs.timer
@@ -0,0 +1,13 @@
+[Unit]
+Description=Store worker logs (timer)
+After=autopkgtest.target
+
+[Timer]
+OnCalendar=00:00
+OnCalendar=06:00
+OnCalendar=12:00
+OnCalendar=18:00
+Unit=store-worker-logs.service
+
+[Install]
+WantedBy=autopkgtest.target