canonical-ubuntu-qa team mailing list archive

[Tim Andersson <mp+456202@xxxxxxxxxxxxxxxxxx>]

Tim Andersson has proposed merging ~andersson123/autopkgtest-cloud:update-github-jobs-swiftclient-refactor into autopkgtest-cloud:master.

Requested reviews:
  Canonical's Ubuntu QA (canonical-ubuntu-qa)

For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/456202
-- 
Your team Canonical's Ubuntu QA is requested to review the proposed merge of ~andersson123/autopkgtest-cloud:update-github-jobs-swiftclient-refactor into autopkgtest-cloud:master.

diff --git a/charms/focal/autopkgtest-web/config.yaml b/charms/focal/autopkgtest-web/config.yaml
index cc97824..9a3148a 100644
--- a/charms/focal/autopkgtest-web/config.yaml
+++ b/charms/focal/autopkgtest-web/config.yaml
@@ -34,3 +34,7 @@ options:
     default:
     description: "$no_proxy environment variable (for accessing sites \
                   other than github, like login.ubuntu.com and launchpad.net)"
+  openstack-creds:
+    type: string
+    default:
+    description: "creds for openstack swift storage for updating github jobs"
diff --git a/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py b/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
index 842348b..5b9e6fd 100644
--- a/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
+++ b/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
@@ -24,6 +24,8 @@ SWIFT_WEB_CREDENTIALS_PATH = os.path.expanduser(
     "~ubuntu/swift-web-credentials.conf"
 )
 
+OPENSTACK_CREDS_PATH = os.path.expanduser("~ubuntu/openstack-creds")
+
 
 @when_not("autopkgtest-web.autopkgtest_web_symlinked")
 def symlink_autopkgtest_cloud():
@@ -259,6 +261,14 @@ def write_swift_web_credentials():
         pass
 
 
+@when_all("config.changed.openstack-creds", "config.set.openstack-creds")
+def write_openstack_creds():
+    openstack_creds = config().get("openstack-creds")
+
+    with open(OPENSTACK_CREDS_PATH, "w") as f:
+        f.write(openstack_creds)
+
+
 @when_not("config.set.swift-web-credentials")
 def clear_swift_web_credentials():
     try:
diff --git a/charms/focal/autopkgtest-web/units/update-github-jobs.service b/charms/focal/autopkgtest-web/units/update-github-jobs.service
index a52bac5..17fb673 100644
--- a/charms/focal/autopkgtest-web/units/update-github-jobs.service
+++ b/charms/focal/autopkgtest-web/units/update-github-jobs.service
@@ -5,6 +5,5 @@ Description=Update GitHub job status
 Type=oneshot
 User=www-data
 Group=www-data
-TimeoutStartSec=1m
-EnvironmentFile=/etc/environment.d/*.conf
+EnvironmentFile=/home/ubuntu/openstack-creds
 ExecStart=/home/ubuntu/webcontrol/update-github-jobs
diff --git a/charms/focal/autopkgtest-web/webcontrol/update-github-jobs b/charms/focal/autopkgtest-web/webcontrol/update-github-jobs
index 6362803..0223385 100755
--- a/charms/focal/autopkgtest-web/webcontrol/update-github-jobs
+++ b/charms/focal/autopkgtest-web/webcontrol/update-github-jobs
@@ -7,40 +7,22 @@ import logging
 import os
 import sys
 import tarfile
-import time
-import urllib.parse
-import urllib.request
-from urllib.error import HTTPError
 
+import swiftclient
 from request.submit import Submit
 
 PENDING_DIR = "/run/autopkgtest_webcontrol/github-pending"
-swift_url = None
-external_url = None
-
-
-def result_matches_job(result_url, params):
-    # download result.tar and get exit code and testinfo
-    for _ in range(5):
-        try:
-            with urllib.request.urlopen(result_url + "/result.tar") as f:
-                tar_bytes = io.BytesIO(f.read())
-            break
-        except IOError as e:
-            logging.debug(
-                "failed to download result %s: %s, retrying...", result_url, e
-            )
-            time.sleep(1)
-    else:
-        logging.error("failed to download result %s", result_url)
-        return
 
+
+def result_matches_job(swift_conn, container, result_obj, params):
+    _, contents = swift_conn.get_object(container, result_obj)
+    tar_bytes = io.BytesIO(contents)
     try:
         with tarfile.open(None, "r", tar_bytes) as tar:
             exitcode = int(tar.extractfile("exitcode").read().strip())
             info = json.loads(tar.extractfile("testinfo.json").read().decode())
     except (KeyError, ValueError, tarfile.TarError) as e:
-        logging.error("broken result %s: %s", result_url, e)
+        logging.error("broken result %s: %s", result_obj, e)
         return
     try:
         result_env = info["custom_environment"]
@@ -88,19 +70,25 @@ def finish_job(jobfile, params, code, log_url):
     Submit.post_json(
         statuses_url,
         data,
-        os.path.expanduser("~/github-status-credentials.txt"),
+        os.path.expanduser("/home/ubuntu/github-status-credentials.txt"),
         params["package"],
     )
 
-    logging.debug("removing job file")
-    os.unlink(jobfile)
+    logging.debug("removing job file %s" % jobfile)
+    try:
+        os.unlink(jobfile)
+    except FileNotFoundError as _:
+        logging.debug(
+            "jobfile %s not found, maybe it was already deleted?" % jobfile
+        )
+        return
+    logging.debug("job %s finished!" % jobfile)
 
 
-def process_job(jobfile):
+def process_job(jobfile, swift_conn, ext_url):
     try:
         with open(jobfile) as f:
             params = json.load(f)
-            mtime = os.fstat(f.fileno()).st_mtime
     except json.decoder.JSONDecodeError as e:
         logging.error("couldn't read %s, skipping: %s", jobfile, e)
         return
@@ -117,47 +105,17 @@ def process_job(jobfile):
         container += "-" + params["ppas"][-1].replace("/", "-")
     except (KeyError, IndexError):
         pass
-    container_url = os.path.join(swift_url, container)
-    package = params["package"]
-    pkghash = package.startswith("lib") and package[:4] or package[0]
-    timestamp = time.strftime("%Y%m%d_%H%M%S", time.gmtime(mtime))
-
-    args = {
-        "format": "plain",
-        "delimiter": "@",
-        "prefix": "%s/%s/%s/%s/"
-        % (params["release"], params["arch"], pkghash, package),
-    }
-    args["marker"] = args["prefix"] + timestamp
-    query_url = container_url + "?" + urllib.parse.urlencode(args)
-    logging.debug("Swift URL query: %s", query_url)
-    try:
-        with urllib.request.urlopen(query_url) as f:
-            for result in f:
-                result_url = os.path.join(
-                    container_url, result.strip().decode()
-                )
-                logging.debug(
-                    "checking result %s for job %s",
-                    result_url,
-                    os.path.basename(jobfile),
-                )
-                code = result_matches_job(result_url, params)
-                if code is not None:
-                    finish_job(
-                        jobfile,
-                        params,
-                        code,
-                        result_url.replace(swift_url, external_url)
-                        + "/log.gz",
-                    )
-                    break
-    except HTTPError as e:
-        logging.error(
-            "job %s URL %s failed: %s", os.path.basename(jobfile), query_url, e
-        )
-        if e.code == 404:
-            os.unlink(jobfile)
+
+    _, object_list = swift_conn.get_container(container, full_listing=True)
+
+    for obj in object_list:
+        if "result.tar" in obj["name"]:
+            code = result_matches_job(
+                swift_conn, container, obj["name"], params
+            )
+            if code is not None:
+                log_url = "/".join([ext_url, container, obj["name"]])
+                finish_job(jobfile, params, code, log_url)
 
 
 if __name__ == "__main__":
@@ -168,12 +126,23 @@ if __name__ == "__main__":
         sys.exit(0)
 
     config = configparser.ConfigParser()
-    config.read(os.path.expanduser("~ubuntu/autopkgtest-cloud.conf"))
-    swift_url = config["web"]["SwiftURL"]
-    try:
-        external_url = config["web"]["ExternalURL"]
-    except KeyError:
-        external_url = swift_url
+    config.read(os.path.expanduser("/home/ubuntu/autopkgtest-cloud.conf"))
+    external_url = config["web"]["ExternalURL"]
+
+    # setup_swift_conn
+    swift_creds = {
+        "authurl": os.environ["OS_AUTH_URL"],
+        "user": os.environ["OS_USERNAME"],
+        "key": os.environ["OS_PASSWORD"],
+        "os_options": {
+            "region_name": os.environ["OS_REGION_NAME"],
+            "project_domain_name": os.environ["OS_PROJECT_DOMAIN_NAME"],
+            "project_name": os.environ["OS_PROJECT_NAME"],
+            "user_domain_name": os.environ["OS_USER_DOMAIN_NAME"],
+        },
+        "auth_version": 3,
+    }
+    swift_conn = swiftclient.Connection(**swift_creds)
 
     jobs = sys.argv[1:]
 
@@ -181,4 +150,4 @@ if __name__ == "__main__":
         jobs = os.listdir(PENDING_DIR)
 
     for job in jobs:
-        process_job(os.path.join(PENDING_DIR, job))
+        process_job(os.path.join(PENDING_DIR, job), swift_conn, external_url)
diff --git a/docs/deploying.rst b/docs/deploying.rst
index 85d853f..17102c0 100644
--- a/docs/deploying.rst
+++ b/docs/deploying.rst
@@ -76,6 +76,10 @@ These are all that are required for a devel deployment. For production there
 are more secrets required, see the ``service-bundle`` and charm
 ``config.yaml`` for more information.
 
+``openstack-creds`` is also a required secrets file for production. Run ``load_creds openstack``
+and then ``env | grep OS_``. The variables set here need to be placed into the secrets directory
+with the filename ``openstack-creds``.
+
 Configure
 ---------
 
diff --git a/mojo/service-bundle b/mojo/service-bundle
index 26b74c0..df2ffe7 100644
--- a/mojo/service-bundle
+++ b/mojo/service-bundle
@@ -189,6 +189,7 @@ applications:
             github-secrets: include-file://{{local_dir}}/github-secrets.json
             github-status-credentials: include-file://{{local_dir}}/github-status-credentials.txt
             swift-web-credentials: include-file://{{local_dir}}/swift-web-credentials.conf
+            openstack-creds: include-file://{{local_dir}}/openstack-creds
             https-proxy: {{ https_proxy }}
             no-proxy: {{ no_proxy }}
 {%- endif %}