canonical-ubuntu-qa team mailing list archive

[Tim Andersson <mp+464740@xxxxxxxxxxxxxxxxxx>]

Tim Andersson has proposed merging ~andersson123/autopkgtest-cloud:make-killing-tests-less-painful into autopkgtest-cloud:master.

Requested reviews:
  Canonical's Ubuntu QA (canonical-ubuntu-qa)

For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/464740
-- 
Your team Canonical's Ubuntu QA is requested to review the proposed merge of ~andersson123/autopkgtest-cloud:make-killing-tests-less-painful into autopkgtest-cloud:master.

diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker
index bfa35e7..40cb8a3 100755
--- a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker
@@ -29,10 +29,14 @@ from urllib.error import HTTPError
 
 import amqplib.client_0_8 as amqp
 import distro_info
+import novaclient.client
+import novaclient.exceptions
 import swiftclient
 import systemd.journal
 from influxdb import InfluxDBClient
 from influxdb.exceptions import InfluxDBClientError
+from keystoneauth1 import session
+from keystoneauth1.identity import v2, v3
 
 ALL_RELEASES = distro_info.UbuntuDistroInfo().get_all(result="object")
 
@@ -621,6 +625,36 @@ def cleanup_and_sleep(out_dir):
     time.sleep(300)
 
 
+def kill_openstack_server(test_uuid: str):
+    if int(os.environ.get("OS_IDENTITY_API_VERSION")) == 3:
+        auth = v3.Password(
+            auth_url=os.environ["OS_AUTH_URL"],
+            username=os.environ["OS_USERNAME"],
+            password=os.environ["OS_PASSWORD"],
+            project_name=os.environ["OS_PROJECT_NAME"],
+            user_domain_name=os.environ["OS_USER_DOMAIN_NAME"],
+            project_domain_name=os.environ["OS_PROJECT_DOMAIN_NAME"],
+        )
+    else:
+        auth = v2.Password(
+            auth_url=os.environ["OS_AUTH_URL"],
+            username=os.environ["OS_USERNAME"],
+            password=os.environ["OS_PASSWORD"],
+            tenant_name=os.environ["OS_TENANT_NAME"],
+        )
+    sess = session.Session(auth=auth)
+    nova = novaclient.client.Client(
+        "2",
+        session=sess,
+        region_name=os.environ["OS_REGION_NAME"],
+    )
+    for instance in nova.servers.list():
+        if test_uuid in instance.name:
+            instance.delete()
+            return instance.name
+    return None
+
+
 def request(msg):
     """Callback for AMQP queue request"""
 
@@ -1118,6 +1152,8 @@ def request(msg):
                     test_uuid,
                     private,
                 )
+                if code == -10:
+                    exit_requested = 99
                 is_failure = code in FAIL_CODES
                 files = set(os.listdir(out_dir))
                 is_unknown_version = "testpkg-version" not in files
@@ -1174,12 +1210,38 @@ def request(msg):
                 elif code == 16 or code < 0:
                     contents = log_contents(out_dir)
                     if exit_requested is not None:
-                        logging.warning(
-                            "Testbed failure and exit %i requested. Log follows:",
-                            exit_requested,
-                        )
-                        logging.error(contents)
-                        sys.exit(exit_requested)
+                        # exit_requested is set to 99 when the test is requested to be killed
+                        if exit_requested != 99:
+                            logging.warning(
+                                "Testbed failure and exit %i requested. Log follows:",
+                                exit_requested,
+                            )
+                            logging.error(contents)
+                            sys.exit(exit_requested)
+                        else:
+                            # Test has been requested to be killed
+                            logging.info(
+                                "Test has been killed by test-killer, exiting."
+                            )
+                            running_test = False
+                            # ack the message so it doesn't go back in the queue
+                            msg.channel.basic_ack(msg.delivery_tag)
+                            # make this a function
+                            logging.info(
+                                "Killing openstack server with uuid %s",
+                                test_uuid,
+                            )
+                            server_name = kill_openstack_server(test_uuid)
+                            if server_name is not None:
+                                logging.info(
+                                    "Deleted test server: %s", server_name
+                                )
+                            else:
+                                logging.info(
+                                    "Failed to delete openstack server: %s"
+                                    % server_name
+                                )
+                            return
                     # Get the package-specific string for triggers too, since they might have broken the run
                     trigs = [
                         t.split("/", 1)[0] for t in params.get("triggers", [])
diff --git a/charms/focal/autopkgtest-web/webcontrol/browse.cgi b/charms/focal/autopkgtest-web/webcontrol/browse.cgi
index 309fb82..85024bf 100755
--- a/charms/focal/autopkgtest-web/webcontrol/browse.cgi
+++ b/charms/focal/autopkgtest-web/webcontrol/browse.cgi
@@ -16,24 +16,13 @@ from helpers.utils import (
     get_all_releases,
     get_autopkgtest_cloud_conf,
     get_supported_releases,
-    setup_key,
+    initialise_app,
 )
-from werkzeug.middleware.proxy_fix import ProxyFix
 
 # Initialize app
-PATH = os.path.join(
-    os.path.sep, os.getenv("XDG_RUNTIME_DIR", "/run"), "autopkgtest_webcontrol"
-)
-os.makedirs(PATH, exist_ok=True)
-app = flask.Flask("browse")
-# we don't want a long cache, as we only serve files that are regularly updated
+PATH, app, secret_path, _ = initialise_app("browse")
 app.config["SEND_FILE_MAX_AGE_DEFAULT"] = 60
 
-app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1)
-
-secret_path = os.path.join(PATH, "secret_key")
-setup_key(app, secret_path)
-
 db_con = None
 swift_container_url = None
 
diff --git a/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py b/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
index 4e26eb8..a92cc51 100644
--- a/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
+++ b/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
@@ -14,12 +14,37 @@ import typing
 
 # introduced in python3.7, we use 3.8
 from dataclasses import dataclass
+from html import escape as _escape
 
 import distro_info
+from flask import Flask
+from flask_openid import OpenID
+from werkzeug.middleware.proxy_fix import ProxyFix
 
 sqlite3.paramstyle = "named"
 
 
+def initialise_app(app_name):
+    PATH = os.path.join(
+        os.path.sep,
+        os.getenv("XDG_RUNTIME_DIR", "/run"),
+        "autopkgtest_webcontrol",
+    )
+    os.makedirs(PATH, exist_ok=True)
+    app = Flask(app_name)
+    app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1)
+    # keep secret persistent between CGI invocations
+    secret_path = os.path.join(PATH, "secret_key")
+    setup_key(app, secret_path)
+    oid = OpenID(app, os.path.join(PATH, "openid"), safe_roots=[])
+    return PATH, app, secret_path, oid
+
+
+def maybe_escape(value):
+    """Escape the value if it is True-ish"""
+    return _escape(value) if value else value
+
+
 @dataclass
 class SqliteWriterConfig:
     writer_exchange_name = "sqlite-write-me.fanout"
@@ -220,3 +245,16 @@ def get_test_id(db_con, release, arch, src):
 
 
 get_test_id._cache = {}
+
+HTML = """
+<!doctype html>
+<html>
+<head>
+<meta charset="utf-8">
+<title>Autopkgtest Test Request</title>
+</head>
+<body>
+{}
+</body>
+</html>
+"""
diff --git a/charms/focal/autopkgtest-web/webcontrol/request/app.py b/charms/focal/autopkgtest-web/webcontrol/request/app.py
index 4fca679..8ee33d4 100644
--- a/charms/focal/autopkgtest-web/webcontrol/request/app.py
+++ b/charms/focal/autopkgtest-web/webcontrol/request/app.py
@@ -5,33 +5,17 @@ import logging
 import os
 import pathlib
 from collections import ChainMap
-from html import escape as _escape
 
-from flask import Flask, redirect, request, session
-from flask_openid import OpenID
+from flask import redirect, request, session
 from helpers.exceptions import WebControlException
-from helpers.utils import setup_key
+from helpers.utils import HTML, initialise_app, maybe_escape
 from request.submit import Submit
-from werkzeug.middleware.proxy_fix import ProxyFix
 
 # map multiple GET vars to AMQP JSON request parameter list
 MULTI_ARGS = {"trigger": "triggers", "ppa": "ppas", "env": "env"}
 
 EMPTY = ""
 
-HTML = """
-<!doctype html>
-<html>
-<head>
-<meta charset="utf-8">
-<title>Autopkgtest Test Request</title>
-</head>
-<body>
-{}
-</body>
-</html>
-"""
-
 LOGIN = """
 <form action="/login" method="post">
 <input type="submit" value="Log in with Ubuntu SSO">
@@ -106,11 +90,6 @@ def invalid(inv_exception, code=400):
     return HTML.format(html), code
 
 
-def maybe_escape(value):
-    """Escape the value if it is True-ish"""
-    return _escape(value) if value else value
-
-
 def get_api_keys():
     """
     API keys is a json file like this:
@@ -132,17 +111,7 @@ def get_api_keys():
 
 
 # Initialize app
-PATH = os.path.join(
-    os.path.sep, os.getenv("XDG_RUNTIME_DIR", "/run"), "autopkgtest_webcontrol"
-)
-os.makedirs(PATH, exist_ok=True)
-app = Flask("request")
-app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1)
-# keep secret persistent between CGI invocations
-secret_path = os.path.join(PATH, "secret_key")
-setup_key(app, secret_path)
-oid = OpenID(app, os.path.join(PATH, "openid"), safe_roots=[])
-
+PATH, app, secret_path, oid = initialise_app("request")
 
 #
 # Flask routes