canonical-ubuntu-qa team mailing list archive

[Tim Andersson <mp+456202@xxxxxxxxxxxxxxxxxx>]

Tim Andersson has proposed merging ~andersson123/autopkgtest-cloud:update-github-jobs-swiftclient-refactor into autopkgtest-cloud:master.

Requested reviews:
  Canonical's Ubuntu QA (canonical-ubuntu-qa)

For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/456202
-- 
Your team Canonical's Ubuntu QA is requested to review the proposed merge of ~andersson123/autopkgtest-cloud:update-github-jobs-swiftclient-refactor into autopkgtest-cloud:master.

diff --git a/charms/focal/autopkgtest-web/webcontrol/update-github-jobs-2 b/charms/focal/autopkgtest-web/webcontrol/update-github-jobs-2
new file mode 100644
index 0000000..9c90daf
--- /dev/null
+++ b/charms/focal/autopkgtest-web/webcontrol/update-github-jobs-2
@@ -0,0 +1,154 @@
+#!/usr/bin/python3
+
+import configparser
+import io
+import json
+import logging
+import os
+import sys
+import tarfile
+import swiftclient
+
+from request.submit import Submit
+
+
+PENDING_DIR = "/run/autopkgtest_webcontrol/github-pending"
+# add this to unit file instead?
+# SECRETS_FILE = "/home/ubuntu/openstack_creds"
+
+
+def result_matches_job(swift_conn, container, result_obj, params):
+    _, contents = swift_conn.get_object(container, result_obj)
+    tar_bytes = io.BytesIO(contents)
+    try:
+        with tarfile.open(None, "r", tar_bytes) as tar:
+            exitcode = int(tar.extractfile("exitcode").read().strip())
+            info = json.loads(tar.extractfile("testinfo.json").read().decode())
+    except (KeyError, ValueError, tarfile.TarError) as e:
+        logging.error("broken result %s: %s", result_obj, e)
+        return
+    try:
+        result_env = info["custom_environment"]
+    except KeyError:
+        logging.info("result has no custom_environment, ignoring")
+        return
+
+    # if the test result has the same parameters than the job, we have a winner
+    if result_env != params["env"]:
+        logging.debug(
+            "exit code: %i, ignoring due to different test env: %s",
+            exitcode,
+            result_env,
+        )
+        return
+
+    logging.debug(
+        "exit code: %i, test env matches job: %s", exitcode, result_env
+    )
+    return exitcode
+
+
+def finish_job(jobfile, params, code, log_url):
+    """Tell GitHub that job is complete and delete the job file"""
+
+    if code in (0, 2):
+        state = "success"
+    elif code in (4, 6, 12):
+        state = "failure"
+    else:
+        state = "error"
+
+    data = {
+        "state": state,
+        "context": "%s-%s" % (params["release"], params["arch"]),
+        "description": "autopkgtest finished (%s)" % state,
+        "target_url": log_url,
+    }
+
+    # find status URL
+    for e in params["env"]:
+        if e.startswith("GITHUB_STATUSES_URL="):
+            statuses_url = e.split("=", 1)[1]
+    # tell GitHub about the result
+    Submit.post_json(
+        statuses_url,
+        data,
+        os.path.expanduser("~/github-status-credentials.txt"),
+        params["package"],
+    )
+
+    logging.debug("removing job file")
+    os.unlink(jobfile)
+    print("job %s finished!" % jobfile)
+
+
+def process_job(jobfile, swift_conn, ext_url):
+    try:
+        with open(jobfile) as f:
+            params = json.load(f)
+            mtime = os.fstat(f.fileno()).st_mtime
+    except json.decoder.JSONDecodeError as e:
+        logging.error("couldn't read %s, skipping: %s", jobfile, e)
+        return
+
+    logging.debug(
+        "\n\n--------------------\nprocessing job %s:\n   %s",
+        os.path.basename(jobfile),
+        params,
+    )
+
+    # fetch Swift results for this request
+    container = "autopkgtest-" + params["release"]
+    try:
+        container += "-" + params["ppas"][-1].replace("/", "-")
+    except (KeyError, IndexError):
+        pass
+
+    cont, object_list = swift_conn.get_container(container, full_listing=True)
+    print(swift_conn.get_auth())
+    print(cont)
+
+    for obj in object_list:
+        if "result.tar" in obj["name"]:
+            code = result_matches_job(swift_conn, container, obj["name"], params)
+            if code is not None:
+                log_url = "/".join([ext_url, container, obj["name"]])
+                finish_job(jobfile, params, code, log_url)
+
+
+if __name__ == "__main__":
+    if "DEBUG" in os.environ:
+        logging.basicConfig(level="DEBUG")
+    if not os.path.isdir(PENDING_DIR):
+        logging.info("%s does not exist, nothing to do", PENDING_DIR)
+        sys.exit(0)
+
+    ###############################################
+    # Need to use openstack creds as env
+    # similar to line 1481 in worker
+    config = configparser.ConfigParser()
+    config.read(os.path.expanduser("~ubuntu/autopkgtest-cloud.conf"))
+    external_url = config["web"]["ExternalURL"]
+
+    # setup_swift_conn
+    swift_creds = {
+        "authurl": os.environ["OS_AUTH_URL"],
+        "user": os.environ["OS_USERNAME"],
+        "key": os.environ["OS_PASSWORD"],
+        "os_options": {
+            "region_name": os.environ["OS_REGION_NAME"],
+            "project_domain_name": os.environ["OS_PROJECT_DOMAIN_NAME"],
+            "project_name": os.environ["OS_PROJECT_NAME"],
+            "user_domain_name": os.environ["OS_USER_DOMAIN_NAME"],
+        },
+        "auth_version": 3,
+    }
+    swift_conn = swiftclient.Connection(**swift_creds)
+
+    jobs = sys.argv[1:]
+
+    if not jobs:
+        jobs = os.listdir(PENDING_DIR)
+
+    for job in jobs:
+        process_job(os.path.join(PENDING_DIR, job), swift_conn, external_url)