canonical-ubuntu-qa team mailing list archive
-
canonical-ubuntu-qa team
-
Mailing list archive
-
Message #01784
[Merge] ~andersson123/autopkgtest-cloud:option_to_stop_test into autopkgtest-cloud:master
Tim Andersson has proposed merging ~andersson123/autopkgtest-cloud:option_to_stop_test into autopkgtest-cloud:master.
Requested reviews:
Canonical's Ubuntu QA (canonical-ubuntu-qa)
For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/453333
--
Your team Canonical's Ubuntu QA is requested to review the proposed merge of ~andersson123/autopkgtest-cloud:option_to_stop_test into autopkgtest-cloud:master.
diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/test-killer.py b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/test-killer.py
new file mode 100755
index 0000000..a69a2a4
--- /dev/null
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/test-killer.py
@@ -0,0 +1,137 @@
+#!/usr/bin/python3
+
+import subprocess
+import time
+
+from flask import Flask, request
+
+app = Flask(__name__)
+
+
+def validate_data(req_json):
+ if not set(["arch", "package", "submit-time"]).issubset(
+ set(req_json.keys())
+ ):
+ return False
+ if not "triggers" in req_json.keys() and not "ppa" in req_json.keys():
+ return False
+ return True
+
+
+def kill_process(killme):
+ if len(killme["responses"]) < 1:
+ return {
+ "state": "failure",
+ "reason": "No matching processes found.",
+ }
+ killme = killme["responses"][0]
+ kill_cmd = ["kill", killme["pid"], "-15"]
+ kill_ps = subprocess.Popen(kill_cmd)
+ kill_ps.wait()
+ time.sleep(10)
+ processes = subprocess.Popen(["ps", "aux"], stdout=subprocess.PIPE)
+ refined_processes = subprocess.check_output(
+ ["grep", "runner"], stdin=processes.stdout
+ )
+ processes.wait()
+ results = refined_processes.splitlines()
+ for l in results:
+ line = str(l)
+ pid = " ".join(line.split()).split(" ")[1]
+ if pid == killme["pid"]:
+ return {
+ "state": "failed",
+ "reason": "process not killed successfully",
+ }
+ return {"state": "success", "processes": []}
+
+
+@app.route("/processes", methods=["GET", "POST"])
+def get_matching_pid():
+ details = request.json
+ processes = subprocess.Popen(["ps", "aux"], stdout=subprocess.PIPE)
+ refined_processes = subprocess.check_output(
+ ["grep", "runner"], stdin=processes.stdout
+ )
+ processes.wait()
+
+ # parse
+ pid_response = {"responses": []}
+ results = refined_processes.splitlines()
+ for line in results:
+ arch = ""
+ triggers = []
+ pkg = ""
+ timestamp = ""
+ res = str(line)
+ pid = " ".join(res.split()).split(" ")[1]
+ # get triggers
+ trig = res
+ while trig.find("ADT_TEST_TRIGGERS=") != -1:
+ trig = trig[
+ trig.find("ADT_TEST_TRIGGERS=") + len("ADT_TEST_TRIGGERS=") :
+ ]
+ this_trigger = trig[: trig.find(" ")]
+ triggers.append(this_trigger)
+ # get arch
+ if res.find("--image ") != -1:
+ image = res[res.find("--image ") + len("--image ") :]
+ image = image[: image.find(" ")]
+ # check for release
+ release = details["release"] if details["release"] in image else ""
+ arch = image.split("-")[2]
+
+ # get timestamp
+ if res.find("--name ") != -1:
+ server_name = res[res.find("--name ") + len("--name ") :]
+ server_name = server_name[: server_name.find(" ")]
+ timestamp = server_name[: server_name.find("juju")]
+ timestamp = timestamp.split("-")
+
+ # get package
+ pkg = "-".join(timestamp[3:-3])
+
+ timestamp = "-".join(timestamp[-3:])[:-1]
+ timestamp = "".join(ch for ch in timestamp if ch.isdigit())
+
+ if (
+ triggers == details["triggers"]
+ # and arch == details["arch"]
+ and pkg == details["package"]
+ and release == details["release"]
+ ):
+ timestamps = [int(timestamp), int(details["submit-time"])]
+ if max(timestamps) - min(timestamps) < 5:
+ pid_response["responses"].append(
+ {
+ "pid": pid,
+ "triggers": triggers,
+ "arch": arch,
+ "submit-time": timestamp,
+ "package": pkg,
+ "command_line": res,
+ "release": release,
+ "requester": details["requester"],
+ }
+ )
+
+ if len(pid_response["responses"]) == 1:
+ return kill_process(pid_response)
+ else:
+ if "pid" in details:
+ for response in pid_response["responses"]:
+ if response["pid"] == details["pid"]:
+ this_response = {
+ "responses": [response],
+ }
+ return kill_process(this_response)
+ else:
+ return {
+ "state": "multiple"
+ if len(pid_response["responses"]) > 0
+ else "none",
+ "processes": pid_response,
+ }
+
+
+app.run(host="0.0.0.0")
diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker
index 980d63b..a3539df 100755
--- a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker
@@ -106,6 +106,8 @@ TEMPORARY_TEST_FAIL_STRINGS = [
" has modification time ",
] # clock skew, LP: #1880839
+CANCELLED_STRING = "Received signal 15"
+
# If we repeatedly time out when installing, there's probably a problem with
# one of the packages' maintainer scripts.
FAIL_STRINGS_REGEX = [
@@ -1076,6 +1078,31 @@ def request(msg):
is_unknown_version = "testpkg-version" not in files
retrying = "Retrying in 5 minutes... " if retry < 2 else ""
+ if CANCELLED_STRING in log_contents(out_dir):
+ contents = log_contents(out_dir)
+ # Check to see if exit was requested with a kill signal
+ # these will be written later on
+ code = 20
+ duration = 0
+ logging.info(
+ "Test run requested to be cancelled, exiting."
+ )
+ # fake a log file
+ with open(os.path.join(out_dir, "log"), "w") as log:
+ log.write("Test requested to be cancelled, exiting.")
+ with open(
+ os.path.join(out_dir, "testpkg-version"), "w"
+ ) as testpkg_version:
+ testpkg_version.write("Test cancelled")
+ dont_run = True
+ submit_metric(
+ architecture,
+ code,
+ pkgname,
+ current_region,
+ False,
+ release,
+ )
if is_failure and is_unknown_version and retry < 2:
# this is an 'unknown' result; try three times but fail
diff --git a/charms/focal/autopkgtest-cloud-worker/layer.yaml b/charms/focal/autopkgtest-cloud-worker/layer.yaml
index 4aeb04d..624e0c4 100644
--- a/charms/focal/autopkgtest-cloud-worker/layer.yaml
+++ b/charms/focal/autopkgtest-cloud-worker/layer.yaml
@@ -24,6 +24,7 @@ options:
- python3-amqplib
- python3-debian
- python3-distro-info
+ - python3-flask
- python3-glanceclient
- python3-influxdb
- python3-keystoneauth1
diff --git a/charms/focal/autopkgtest-cloud-worker/units/test-killer.service b/charms/focal/autopkgtest-cloud-worker/units/test-killer.service
new file mode 100644
index 0000000..dac0550
--- /dev/null
+++ b/charms/focal/autopkgtest-cloud-worker/units/test-killer.service
@@ -0,0 +1,13 @@
+[Unit]
+Description=HTTP endpoint for killing autopkgtests
+StartLimitIntervalSec=60s
+StartLimitBurst=60
+
+[Service]
+User=ubuntu
+ExecStart=/home/ubuntu/autopkgtest-cloud/tools/test-killer.py
+Restart=on-failure
+RestartSec=1s
+
+[Install]
+WantedBy=autopkgtest.target
diff --git a/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py b/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
index 842348b..36e3b55 100644
--- a/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
+++ b/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
@@ -195,6 +195,7 @@ def set_up_web_config(apache):
# webcontrol CGI scripts
ScriptAlias /request.cgi {webcontrol_dir}/request.cgi/
+ ScriptAlias /stop.cgi {webcontrol_dir}/stop.cgi/
ScriptAlias /login {webcontrol_dir}/request.cgi/login
ScriptAlias /logout {webcontrol_dir}/request.cgi/logout
ScriptAlias /private-results {webcontrol_dir}/private-results.cgi/
diff --git a/charms/focal/autopkgtest-web/webcontrol/browse.cgi b/charms/focal/autopkgtest-web/webcontrol/browse.cgi
index d7bc343..edaeb31 100755
--- a/charms/focal/autopkgtest-web/webcontrol/browse.cgi
+++ b/charms/focal/autopkgtest-web/webcontrol/browse.cgi
@@ -358,6 +358,27 @@ def running():
running_info = json.load(f)
except FileNotFoundError:
running_info = {}
+
+ for pkg in running_info.keys():
+ for runhash in running_info[pkg].keys():
+ for release in running_info[pkg][runhash].keys():
+ for arch in running_info[pkg][runhash][release].keys():
+ env_str = ""
+ for k, v in running_info[pkg][runhash][release][arch][0].items():
+ if k == "submit-time":
+ val = ''.join(ch for ch in v if ch.isdigit())
+ env_str += "&" + str(k) + "=" + str(val)
+ elif k == "triggers":
+ val = ""
+ for trig in v:
+ val += "&trigger=" + trig
+ env_str += val
+ else:
+ val = v
+ env_str += "&" + str(k) + "=" + str(val)
+ running_info[pkg][runhash][release][arch].append(
+ env_str
+ )
return render(
"browse-running.html",
diff --git a/charms/focal/autopkgtest-web/webcontrol/stop.cgi b/charms/focal/autopkgtest-web/webcontrol/stop.cgi
new file mode 100755
index 0000000..cd4ba19
--- /dev/null
+++ b/charms/focal/autopkgtest-web/webcontrol/stop.cgi
@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+
+"""Run request app as CGI script """
+
+from wsgiref.handlers import CGIHandler
+
+from stop.app import app
+
+if __name__ == "__main__":
+ app.config["DEBUG"] = True
+ CGIHandler().run(app)
diff --git a/charms/focal/autopkgtest-web/webcontrol/stop/__init__.py b/charms/focal/autopkgtest-web/webcontrol/stop/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/charms/focal/autopkgtest-web/webcontrol/stop/__init__.py
diff --git a/charms/focal/autopkgtest-web/webcontrol/stop/app.py b/charms/focal/autopkgtest-web/webcontrol/stop/app.py
new file mode 100644
index 0000000..3ca6d83
--- /dev/null
+++ b/charms/focal/autopkgtest-web/webcontrol/stop/app.py
@@ -0,0 +1,190 @@
+"""Stop Request Flask App"""
+import json
+import os
+import urllib
+from html import escape as _escape
+from urllib.error import HTTPError
+
+import requests
+from flask import Flask, request, session
+from flask_openid import OpenID
+from helpers.utils import setup_key
+from werkzeug.middleware.proxy_fix import ProxyFix
+
+# map multiple GET vars to AMQP JSON request parameter list
+MULTI_ARGS = {"trigger": "triggers", "ppa": "ppas", "env": "env"}
+
+LP = "https://api.launchpad.net/1.0/";
+
+ALLOWED_TEAMS = [
+ "canonical-ubuntu-qa",
+ "canonical-foundations",
+]
+
+HTML = """
+<!doctype html>
+<html>
+<head>
+<meta charset="utf-8">
+<title>Autopkgtest Test Request</title>
+</head>
+<body>
+{}
+</body>
+</html>
+"""
+
+
+def in_allowed_team_for_stop(requester):
+ _, response = lp_request(
+ "~%s/memberships_details?ws.size=300" % requester, {}
+ )
+ entries = response.get("entries")
+ for e in entries:
+ for team in ALLOWED_TEAMS:
+ if team in e["team_link"]:
+ return True
+ return False
+
+
+def lp_request(obj, query):
+ """Do a Launchpad REST request
+
+ Request https://api.launchpad.net/1.0/<obj>?<query>.
+
+ Return (code, json), where json is defined for successful codes
+ (200 <= code < 300) and None otherwise.
+ """
+ url = LP + obj + "?" + urllib.parse.urlencode(query)
+ try:
+ with urllib.request.urlopen(url, timeout=10) as req:
+ code = req.getcode()
+ if code >= 300:
+ # logging.error(
+ # "URL %s failed with code %u", req.geturl(), code
+ # )
+ return (code, None)
+ response = req.read()
+ except HTTPError as e:
+ # logging.error(
+ # "%s failed with %u: %s\n%s", url, e.code, e.reason, e.headers
+ # )
+ return (e.code, None)
+
+ try:
+ response = json.loads(response.decode("UTF-8"))
+ # except (UnicodeDecodeError, ValueError) as e:
+ except (UnicodeDecodeError, ValueError) as _:
+ # logging.error(
+ # "URL %s gave invalid response %s: %s",
+ # req.geturl(),
+ # response,
+ # str(e),
+ # )
+ return (500, None)
+ # logging.debug("lp_request %s succeeded: %s", url, response)
+ return (code, response)
+
+
+def get_autopkgtest_cloud_worker_ips():
+ ipstr = ""
+ with open("/home/ubuntu/cloud-worker-ips", "r") as f:
+ ipstr = f.read()
+ return ipstr.splitlines()
+
+
+def maybe_escape(value):
+ """Escape the value if it is True-ish"""
+ return _escape(value) if value else value
+
+
+# Initialize app
+PATH = os.path.join(
+ os.path.sep, os.getenv("XDG_RUNTIME_DIR", "/run"), "autopkgtest_webcontrol"
+)
+os.makedirs(PATH, exist_ok=True)
+app = Flask("stop")
+app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1)
+# keep secret persistent between CGI invocations
+secret_path = os.path.join(PATH, "secret_key")
+setup_key(app, secret_path)
+oid = OpenID(app, os.path.join(PATH, "openid"), safe_roots=[])
+
+
+@app.route("/", methods=["GET", "POST"])
+def index_root():
+ session.permanent = True
+ params = {
+ maybe_escape(k): maybe_escape(v) for k, v in request.args.items()
+ }
+ # convert multiple GET args into lists
+ for getarg, paramname in MULTI_ARGS.items():
+ try:
+ del params[getarg]
+ except KeyError:
+ pass
+ l = request.args.getlist(getarg)
+ if l:
+ params[paramname] = [maybe_escape(p) for p in l]
+
+ if "nickname" not in session:
+ return (
+ HTML.format(
+ "<p>Please log in to stop a test with the /login endpoint.</p>"
+ ),
+ 400,
+ )
+ else:
+ if not in_allowed_team_for_stop(session["nickname"]):
+ return (
+ HTML.format(
+ "<p>You do not have the necessary permissions to stop a test.</p>"
+ ),
+ 400,
+ )
+
+ ips = get_autopkgtest_cloud_worker_ips()
+ matching_processes = []
+ k = 0
+ for ip in ips:
+ res = requests.post("http://"; + ip + ":5000/processes", json=params)
+ result = res.json()
+ if "state" in result.keys() and result["state"] == "success":
+ return HTML.format("<p>Test successfully killed.</p>"), 200
+ elif "state" in result.keys() and result["state"] == "multiple":
+ result["ip"] = k
+ matching_processes.append(result)
+ elif "state" in result.keys() and result["state"] == "failed":
+ return (
+ HTML.format(
+ "<p>Test couldn't be killed, please contact a member of the Ubuntu QA team.</p>"
+ ),
+ 400,
+ )
+ else:
+ return HTML.format("<p>Something unexpected happened.</p>"), 400
+ k += 1
+ resp_str = ""
+ for response in matching_processes:
+ for respon in response["processes"]["responses"]:
+ url = "stop.cgi?"
+ param_lst = []
+ for kwarg in respon.keys():
+ if kwarg == "triggers":
+ for trig in respon["triggers"]:
+ param_lst.append("trigger=" + trig)
+ elif kwarg != "command_line":
+ param_lst.append(kwarg + "=" + respon[kwarg])
+ else:
+ pass
+ url += "&".join(param_lst)
+ resp_str += (
+ """<button onclick="location.href='"""
+ + url
+ + """'" type="button">Kill the test below</button>"""
+ )
+
+ json_resp = json.dumps(respon, indent=2)
+ for line in json_resp.splitlines():
+ resp_str += "<p>" + line + "</p>"
+ return HTML.format(resp_str), 400
diff --git a/charms/focal/autopkgtest-web/webcontrol/templates/browse-running.html b/charms/focal/autopkgtest-web/webcontrol/templates/browse-running.html
index 23df383..18621ce 100644
--- a/charms/focal/autopkgtest-web/webcontrol/templates/browse-running.html
+++ b/charms/focal/autopkgtest-web/webcontrol/templates/browse-running.html
@@ -40,7 +40,7 @@
<h2 id="pkg-{{p}}"><a href="/packages/{{p}}">{{p}}</a></h2>
{% for runhash, relinfo in running[p].items() %}
{% for release, archinfo in relinfo.items() %}
- {% for arch, (params, duration, logtail) in archinfo.items() %}
+ {% for arch, (params, duration, logtail, param_string) in archinfo.items() %}
<table class="table-condensed">
<tr><th>Release:</th><td>{{release}}</td></tr>
<tr><th>Architecture:</th><td>{{arch}}</td></tr>
@@ -48,6 +48,7 @@
<tr><th>{{param|capitalize}}:</th><td>{{v}}</td></tr>
{% endfor %}
<tr><th>Running for:</th><td>{{duration//3600 }}h {{duration % 3600//60}}m {{duration % 60}}s</td></tr>
+ <tr><a href="{{base_url}}stop.cgi?release={{release}}&arch={{arch}}&package={{p}}{{param_string}}">Stop this test</a></tr>
</table>
<pre>
{{logtail}}
diff --git a/mojo/cloud-worker-ip-to-autopkgtest-web b/mojo/cloud-worker-ip-to-autopkgtest-web
new file mode 100755
index 0000000..3d9c60d
--- /dev/null
+++ b/mojo/cloud-worker-ip-to-autopkgtest-web
@@ -0,0 +1,12 @@
+#!/bin/sh
+# shellcheck disable=SC2086
+
+ips=$(juju status --format=json autopkgtest-cloud-worker | jq --monochrome-output --raw-output '.applications["autopkgtest-cloud-worker"].units | map(.["public-address"])[]')
+
+printf "%s\n" "${ips}" > /tmp/cloud-worker-ips
+
+machines=$(juju status --format=json apache2 | jq --monochrome-output --raw-output '.applications["apache2"].units | map(.["machine"])[]')
+
+for machine in $machines; do
+ juju scp /tmp/cloud-worker-ips $machine:/home/ubuntu/cloud-worker-ips
+done
diff --git a/mojo/manifest b/mojo/manifest
index 75f677e..b499283 100644
--- a/mojo/manifest
+++ b/mojo/manifest
@@ -3,3 +3,4 @@ secrets
bundle config=service-bundle wait=True max-wait=1200
script config=make-lxd-secgroup
script config=postdeploy
+script config=cloud-worker-ip-to-autopkgtest-web
References
