canonical-ubuntu-qa team mailing list archive

[Tim Andersson <mp+453333@xxxxxxxxxxxxxxxxxx>]

Tim Andersson has proposed merging ~andersson123/autopkgtest-cloud:option_to_stop_test into autopkgtest-cloud:master.

Requested reviews:
  Canonical's Ubuntu QA (canonical-ubuntu-qa)

For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/453333
-- 
Your team Canonical's Ubuntu QA is requested to review the proposed merge of ~andersson123/autopkgtest-cloud:option_to_stop_test into autopkgtest-cloud:master.

diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/test-killer.py b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/test-killer.py
new file mode 100755
index 0000000..5543f9c
--- /dev/null
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/test-killer.py
@@ -0,0 +1,154 @@
+#!/usr/bin/python3
+
+# gna try use swift to add a way for autopkgtest-web and autopkgtest-cloud-worker to communicate
+# do dummy endpoint first to check communication works
+
+import subprocess
+import time
+
+from flask import Flask, request
+
+app = Flask(__name__)
+
+
+def validate_data(req_json):
+    if not set(["arch", "package", "submit-time"]).issubset(
+        set(req_json.keys())
+    ):
+        return False
+    if not "triggers" in req_json.keys() and not "ppa" in req_json.keys():
+        return False
+    return True
+
+
+# # Shouldn't be needed
+# @app.route("/kill", methods=["GET", "POST"])
+# def kill_endpoint():
+#     response = kill_process(request.json)
+#     return response
+
+
+def kill_process(killme):
+    # killme = request.json
+    if len(killme["responses"]) < 1:
+        return {
+            "state": "failure",
+            "reason": "No matching processes found.",
+        }
+    killme = killme["responses"][0]
+    kill_cmd = ["kill", killme["pid"], "-15"]
+    kill_ps = subprocess.Popen(kill_cmd)
+    kill_ps.wait()
+    time.sleep(10)
+    processes = subprocess.Popen(["ps", "aux"], stdout=subprocess.PIPE)
+    refined_processes = subprocess.check_output(
+        ["grep", "runner"], stdin=processes.stdout
+    )
+    processes.wait()
+    results = refined_processes.splitlines()
+    for l in results:
+        line = str(l)
+        pid = " ".join(line.split()).split(" ")[1]
+        if pid == killme["pid"]:
+            return {
+                "state": "failed",
+                "reason": "process not killed successfully",
+            }
+    return {"state": "success", "processes": []}
+
+
+@app.route("/processes", methods=["GET", "POST"])
+def get_matching_pid():
+    details = request.json
+    # Need to redo this
+    # if not validate_data(details):
+    #     return ({"failed": "Incorrect args passed"}, 403)
+    processes = subprocess.Popen(["ps", "aux"], stdout=subprocess.PIPE)
+    refined_processes = subprocess.check_output(
+        ["grep", "runner"], stdin=processes.stdout
+    )
+    processes.wait()
+
+    # parse
+    pid_response = {"responses": []}
+    results = refined_processes.splitlines()
+    for line in results:
+        arch = ""
+        triggers = []
+        pkg = ""
+        timestamp = ""
+        res = str(line)
+        pid = " ".join(res.split()).split(" ")[1]
+        # get triggers
+        trig = res
+        while trig.find("ADT_TEST_TRIGGERS=") != -1:
+            trig = trig[
+                trig.find("ADT_TEST_TRIGGERS=") + len("ADT_TEST_TRIGGERS=") :
+            ]
+            this_trigger = trig[: trig.find(" ")]
+            triggers.append(this_trigger)
+        # get arch
+        if res.find("--image ") != -1:
+            image = res[res.find("--image ") + len("--image ") :]
+            image = image[: image.find(" ")]
+            # check for release
+            release = details["release"] if details["release"] in image else ""
+            arch = image.split("-")[2]
+
+        # get timestamp
+        if res.find("--name ") != -1:
+            server_name = res[res.find("--name ") + len("--name ") :]
+            server_name = server_name[: server_name.find(" ")]
+            timestamp = server_name[: server_name.find("juju")]
+            timestamp = timestamp.split("-")
+
+            # get package
+            pkg = "-".join(timestamp[3:-3])
+
+            timestamp = "-".join(timestamp[-3:])[:-1]
+            timestamp = "".join(ch for ch in timestamp if ch.isdigit())
+
+        if (
+            triggers == details["triggers"]
+            # and arch == details["arch"]
+            and pkg == details["package"]
+            and release == details["release"]
+        ):
+            timestamps = [int(timestamp), int(details["submit-time"])]
+            if max(timestamps) - min(timestamps) < 5:
+                pid_response["responses"].append(
+                    {
+                        "pid": pid,
+                        "triggers": triggers,
+                        "arch": arch,
+                        "submit-time": timestamp,
+                        "package": pkg,
+                        "command_line": res,
+                        "release": release,
+                        "requester": details["requester"],
+                    }
+                )
+
+    if len(pid_response["responses"]) == 1:
+        return kill_process(pid_response)
+    else:
+        if "pid" in details:
+            for response in pid_response["responses"]:
+                if response["pid"] == details["pid"]:
+                    this_response = {
+                        "responses": [response],
+                    }
+                    return kill_process(this_response)
+        else:
+            # display the various results - user can select one.
+            # NO PID. just index maybe?
+            # implement for when there's more than one match!
+            return {
+                "state": "multiple"
+                if len(pid_response["responses"]) > 0
+                else "none",
+                "processes": pid_response,
+            }
+
+
+app.run(host="0.0.0.0")
diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker
index 980d63b..e294928 100755
--- a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker
@@ -106,6 +106,8 @@ TEMPORARY_TEST_FAIL_STRINGS = [
     " has modification time ",
 ]  # clock skew, LP: #1880839
 
+CANCELLED_STRING = "Received signal 15"
+
 # If we repeatedly time out when installing, there's probably a problem with
 # one of the packages' maintainer scripts.
 FAIL_STRINGS_REGEX = [
@@ -1076,10 +1078,61 @@ def request(msg):
                 is_unknown_version = "testpkg-version" not in files
 
                 retrying = "Retrying in 5 minutes... " if retry < 2 else ""
+                if CANCELLED_STRING in log_contents(out_dir):
+                    contents = log_contents(out_dir)
+                    # Check to see if exit was requested with a kill signal
+                    # code = 99
+                    submit_metric(
+                        architecture,
+                        code,
+                        pkgname,
+                        current_region,
+                        False,
+                        release,
+                    )
+                    logging.warning(
+                        "Test run requested to be cancelled, exiting."
+                    )
+                    # global amqp_con
+                    # complete_amqp = amqp_con.channel()
+                    # complete_amqp.access_request(
+                    #     "/complete", active=True, read=False, write=True
+                    # )
+                    # complete_amqp.exchange_declare(
+                    #     complete_exchange_name,
+                    #     "fanout",
+                    #     durable=True,
+                    #     auto_delete=False,
+                    # )
+                    # complete_msg = json.dumps(
+                    #     {
+                    #         "architecture": architecture,
+                    #         "container": container,
+                    #         "duration": duration,
+                    #         "exitcode": code,
+                    #         "package": pkgname,
+                    #         "testpkg_version": testpkg_version,
+                    #         "release": release,
+                    #         "requester": requester,
+                    #         "swift_dir": swift_dir,
+                    #         "triggers": triggers,
+                    #     }
+                    # )
+                    # complete_amqp.basic_publish(
+                    #     amqp.Message(complete_msg, delivery_mode=2),
+                    #     complete_exchange_name,
+                    #     "",
+                    # )
+
+                    # logging.info("Acknowledging request %s" % body)
+                    # msg.channel.basic_ack(msg.delivery_tag)
+                    # running_test = False
+                    # sys.exit(0)
 
                 if is_failure and is_unknown_version and retry < 2:
                     # this is an 'unknown' result; try three times but fail
                     # properly after that (do not tmpfail)
+                    # need to charm what's below
                     contents = log_contents(out_dir)
                     logging.warning(
                         "Test run failed with no version. %sLog follows:",
diff --git a/charms/focal/autopkgtest-cloud-worker/layer.yaml b/charms/focal/autopkgtest-cloud-worker/layer.yaml
index 4aeb04d..624e0c4 100644
--- a/charms/focal/autopkgtest-cloud-worker/layer.yaml
+++ b/charms/focal/autopkgtest-cloud-worker/layer.yaml
@@ -24,6 +24,7 @@ options:
       - python3-amqplib
       - python3-debian
       - python3-distro-info
+      - python3-flask
       - python3-glanceclient
       - python3-influxdb
       - python3-keystoneauth1
diff --git a/charms/focal/autopkgtest-cloud-worker/units/test-killer.service b/charms/focal/autopkgtest-cloud-worker/units/test-killer.service
new file mode 100644
index 0000000..dac0550
--- /dev/null
+++ b/charms/focal/autopkgtest-cloud-worker/units/test-killer.service
@@ -0,0 +1,13 @@
+[Unit]
+Description=HTTP endpoint for killing autopkgtests
+StartLimitIntervalSec=60s
+StartLimitBurst=60
+
+[Service]
+User=ubuntu
+ExecStart=/home/ubuntu/autopkgtest-cloud/tools/test-killer.py
+Restart=on-failure
+RestartSec=1s
+
+[Install]
+WantedBy=autopkgtest.target
diff --git a/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py b/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
index 842348b..36e3b55 100644
--- a/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
+++ b/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
@@ -195,6 +195,7 @@ def set_up_web_config(apache):
 
                 # webcontrol CGI scripts
                 ScriptAlias /request.cgi {webcontrol_dir}/request.cgi/
+                ScriptAlias /stop.cgi {webcontrol_dir}/stop.cgi/
                 ScriptAlias /login {webcontrol_dir}/request.cgi/login
                 ScriptAlias /logout {webcontrol_dir}/request.cgi/logout
                 ScriptAlias /private-results {webcontrol_dir}/private-results.cgi/
diff --git a/charms/focal/autopkgtest-web/webcontrol/browse.cgi b/charms/focal/autopkgtest-web/webcontrol/browse.cgi
index d7bc343..edaeb31 100755
--- a/charms/focal/autopkgtest-web/webcontrol/browse.cgi
+++ b/charms/focal/autopkgtest-web/webcontrol/browse.cgi
@@ -358,6 +358,27 @@ def running():
             running_info = json.load(f)
     except FileNotFoundError:
         running_info = {}
+    
+    for pkg in running_info.keys():
+        for runhash in running_info[pkg].keys():
+            for release in running_info[pkg][runhash].keys():
+                for arch in running_info[pkg][runhash][release].keys():
+                    env_str = ""
+                    for k, v in running_info[pkg][runhash][release][arch][0].items():
+                        if k == "submit-time":
+                            val = ''.join(ch for ch in v if ch.isdigit())
+                            env_str += "&" + str(k) + "=" + str(val)
+                        elif k == "triggers":
+                            val = ""
+                            for trig in v:
+                                val += "&trigger=" + trig
+                            env_str += val
+                        else:
+                            val = v
+                            env_str += "&" + str(k) + "=" + str(val)
+                    running_info[pkg][runhash][release][arch].append(
+                        env_str
+                    )
 
     return render(
         "browse-running.html",
diff --git a/charms/focal/autopkgtest-web/webcontrol/stop.cgi b/charms/focal/autopkgtest-web/webcontrol/stop.cgi
new file mode 100755
index 0000000..cd4ba19
--- /dev/null
+++ b/charms/focal/autopkgtest-web/webcontrol/stop.cgi
@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+
+"""Run request app as CGI script """
+
+from wsgiref.handlers import CGIHandler
+
+from stop.app import app
+
+if __name__ == "__main__":
+    app.config["DEBUG"] = True
+    CGIHandler().run(app)
diff --git a/charms/focal/autopkgtest-web/webcontrol/stop/__init__.py b/charms/focal/autopkgtest-web/webcontrol/stop/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/charms/focal/autopkgtest-web/webcontrol/stop/__init__.py
diff --git a/charms/focal/autopkgtest-web/webcontrol/stop/app.py b/charms/focal/autopkgtest-web/webcontrol/stop/app.py
new file mode 100644
index 0000000..866625f
--- /dev/null
+++ b/charms/focal/autopkgtest-web/webcontrol/stop/app.py
@@ -0,0 +1,122 @@
+"""Stop Request Flask App"""
+import json
+import os
+from html import escape as _escape
+
+import requests
+from flask import Flask, request, session
+from flask_openid import OpenID
+from helpers.utils import setup_key
+from werkzeug.middleware.proxy_fix import ProxyFix
+
+# map multiple GET vars to AMQP JSON request parameter list
+MULTI_ARGS = {"trigger": "triggers", "ppa": "ppas", "env": "env"}
+
+HTML = """
+<!doctype html>
+<html>
+<head>
+<meta charset="utf-8">
+<title>Autopkgtest Test Request</title>
+</head>
+<body>
+{}
+</body>
+</html>
+"""
+
+
+def get_autopkgtest_cloud_worker_ips():
+    # implement an actual function here
+    ipstr = ""
+    with open("/home/ubuntu/cloud-worker-ips", "r") as f:
+        ipstr = f.read()
+    return ipstr.splitlines()
+
+
+def maybe_escape(value):
+    """Escape the value if it is True-ish"""
+    return _escape(value) if value else value
+
+
+# Initialize app
+PATH = os.path.join(
+    os.path.sep, os.getenv("XDG_RUNTIME_DIR", "/run"), "autopkgtest_webcontrol"
+)
+os.makedirs(PATH, exist_ok=True)
+app = Flask("stop")
+app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1)
+# keep secret persistent between CGI invocations
+secret_path = os.path.join(PATH, "secret_key")
+setup_key(app, secret_path)
+oid = OpenID(app, os.path.join(PATH, "openid"), safe_roots=[])
+
+
+@app.route("/", methods=["GET", "POST"])
+def index_root():
+    session.permanent = True
+    params = {
+        maybe_escape(k): maybe_escape(v) for k, v in request.args.items()
+    }
+    # convert multiple GET args into lists
+    for getarg, paramname in MULTI_ARGS.items():
+        try:
+            del params[getarg]
+        except KeyError:
+            pass
+        l = request.args.getlist(getarg)
+        if l:
+            params[paramname] = [maybe_escape(p) for p in l]
+    ips = get_autopkgtest_cloud_worker_ips()
+    matching_processes = []
+    k = 0
+    for ip in ips:
+        res = requests.post("http://"; + ip + ":5000/processes", json=params)
+        result = res.json()
+        if "state" in result.keys() and result["state"] == "success":
+            return HTML.format("<p>Test successfully killed.</p>"), 200
+        elif "state" in result.keys() and result["state"] == "multiple":
+            result["ip"] = k
+            matching_processes.append(result)
+        elif "state" in result.keys() and result["state"] == "failed":
+            return (
+                HTML.format(
+                    "<p>Test couldn't be killed, please contact a member of the Ubuntu QA team.</p>"
+                ),
+                400,
+            )
+        else:
+            return HTML.format("<p>Something unexpected happened.</p>"), 400
+        k += 1
+        # <button onclick="location.href='http://www.example.com'" type="button">
+        #  www.example.com</button>
+    # kwargs = ["triggers",
+    # kwargs = ["release",
+    #           "arch",
+    #           "package",
+    #           "submit-time"]
+    resp_str = ""
+    for response in matching_processes:
+        for respon in response["processes"]["responses"]:
+            # https://autopkgtest.staging.ubuntu.com/stop.cgi?release=mantic&arch=arm64&package=gzip&requester=andersson123&submit-time=20231023155411&trigger=gzip/1.12-1ubuntu1
+            url = "stop.cgi?"
+            param_lst = []
+            for kwarg in respon.keys():
+                if kwarg == "triggers":
+                    for trig in respon["triggers"]:
+                        param_lst.append("trigger=" + trig)
+                elif kwarg != "command_line":
+                    param_lst.append(kwarg + "=" + respon[kwarg])
+                else:
+                    pass
+            url += "&".join(param_lst)
+            resp_str += (
+                """<button onclick="location.href='"""
+                + url
+                + """'" type="button">Kill the test below</button>"""
+            )
+
+            json_resp = json.dumps(respon, indent=2)
+            for line in json_resp.splitlines():
+                resp_str += "<p>" + line + "</p>"
+    return HTML.format(resp_str), 400
diff --git a/charms/focal/autopkgtest-web/webcontrol/templates/browse-running.html b/charms/focal/autopkgtest-web/webcontrol/templates/browse-running.html
index 23df383..18621ce 100644
--- a/charms/focal/autopkgtest-web/webcontrol/templates/browse-running.html
+++ b/charms/focal/autopkgtest-web/webcontrol/templates/browse-running.html
@@ -40,7 +40,7 @@
   <h2 id="pkg-{{p}}"><a href="/packages/{{p}}">{{p}}</a></h2>
     {% for runhash, relinfo in running[p].items() %}
       {% for release, archinfo in relinfo.items() %}
-        {% for arch, (params, duration, logtail) in archinfo.items() %}
+        {% for arch, (params, duration, logtail, param_string) in archinfo.items() %}
           <table class="table-condensed">
             <tr><th>Release:</th><td>{{release}}</td></tr>
             <tr><th>Architecture:</th><td>{{arch}}</td></tr>
@@ -48,6 +48,7 @@
             <tr><th>{{param|capitalize}}:</th><td>{{v}}</td></tr>
             {% endfor %}
             <tr><th>Running for:</th><td>{{duration//3600 }}h {{duration % 3600//60}}m {{duration % 60}}s</td></tr>
+            <tr><a href="{{base_url}}stop.cgi?release={{release}}&arch={{arch}}&package={{p}}{{param_string}}">Stop this test</a></tr>
          </table>
         <pre>
 {{logtail}}
diff --git a/mojo/cloud-worker-ip-to-autopkgtest-web b/mojo/cloud-worker-ip-to-autopkgtest-web
new file mode 100755
index 0000000..3d9c60d
--- /dev/null
+++ b/mojo/cloud-worker-ip-to-autopkgtest-web
@@ -0,0 +1,12 @@
+#!/bin/sh
+# shellcheck disable=SC2086
+
+ips=$(juju status --format=json autopkgtest-cloud-worker | jq --monochrome-output --raw-output '.applications["autopkgtest-cloud-worker"].units | map(.["public-address"])[]')
+
+printf "%s\n" "${ips}" > /tmp/cloud-worker-ips
+
+machines=$(juju status --format=json apache2 | jq --monochrome-output --raw-output '.applications["apache2"].units | map(.["machine"])[]')
+
+for machine in $machines; do
+    juju scp /tmp/cloud-worker-ips $machine:/home/ubuntu/cloud-worker-ips
+done
diff --git a/mojo/manifest b/mojo/manifest
index 75f677e..b499283 100644
--- a/mojo/manifest
+++ b/mojo/manifest
@@ -3,3 +3,4 @@ secrets
 bundle config=service-bundle wait=True max-wait=1200
 script config=make-lxd-secgroup
 script config=postdeploy
+script config=cloud-worker-ip-to-autopkgtest-web