canonical-ubuntu-qa team mailing list archive
-
canonical-ubuntu-qa team
-
Mailing list archive
-
Message #03036
[Merge] ~hyask/autopkgtest-cloud:skia/ease_browse_dev into autopkgtest-cloud:master
Skia has proposed merging ~hyask/autopkgtest-cloud:skia/ease_browse_dev into autopkgtest-cloud:master.
Requested reviews:
Canonical's Ubuntu QA (canonical-ubuntu-qa)
For more details, see:
https://code.launchpad.net/~hyask/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/461027
Allow easier local development.
--
Your team Canonical's Ubuntu QA is requested to review the proposed merge of ~hyask/autopkgtest-cloud:skia/ease_browse_dev into autopkgtest-cloud:master.
diff --git a/charms/focal/autopkgtest-web/charmcraft.yaml b/charms/focal/autopkgtest-web/charmcraft.yaml
index 93c0816..546437f 100644
--- a/charms/focal/autopkgtest-web/charmcraft.yaml
+++ b/charms/focal/autopkgtest-web/charmcraft.yaml
@@ -6,6 +6,7 @@ parts:
build-snaps: [charm]
build-packages:
- libjs-jquery
+ - libjs-bootstrap
- python3-dev
bases:
- build-on:
diff --git a/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py b/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
index e1be1a2..6920fc1 100644
--- a/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
+++ b/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
@@ -338,20 +338,6 @@ def clear_github_status_credentials():
pass
-@when_not("autopkgtest-web.bootstrap-symlinked")
-def symlink_bootstrap():
- try:
- os.symlink(
- os.path.join(
- os.path.sep, "usr", "share", "javascript", "bootstrap"
- ),
- os.path.join(charm_dir(), "webcontrol", "static", "bootstrap"),
- )
- set_flag("autopkgtest-web.bootstrap-symlinked")
- except FileExistsError:
- pass
-
-
@when_not("autopkgtest-web.runtime-dir-created")
def make_runtime_tmpfiles():
with open("/etc/tmpfiles.d/autopkgtest-web-runtime.conf", "w") as r:
diff --git a/charms/focal/autopkgtest-web/webcontrol/README.md b/charms/focal/autopkgtest-web/webcontrol/README.md
new file mode 100644
index 0000000..e489c2a
--- /dev/null
+++ b/charms/focal/autopkgtest-web/webcontrol/README.md
@@ -0,0 +1,12 @@
+# autopkgtest-cloud web frontend
+
+## Developing browse.cgi locally
+
+Install the dependencies:
+`sudo apt install python3-flask python3-distro-info libjs-jquery libjs-bootstrap`
+
+Then simply run `./browse-test-py`, it will launch the flask application locally
+with some mocked data.
+As the import of `browse.cgi` is done trough `importlib`, changes in that file
+will not be reloaded automatically, so you'll still need to restart the app
+manually.
diff --git a/charms/focal/autopkgtest-web/webcontrol/browse-test.py b/charms/focal/autopkgtest-web/webcontrol/browse-test.py
new file mode 100755
index 0000000..a5c5b4e
--- /dev/null
+++ b/charms/focal/autopkgtest-web/webcontrol/browse-test.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+"""Run browse app in local debug mode for testing."""
+
+import importlib
+from pathlib import Path
+
+from helpers import tests, utils
+
+# import browse.cgi
+browse_path = str(Path(__file__).parent / "browse.cgi")
+loader = importlib.machinery.SourceFileLoader("browse", browse_path)
+spec = importlib.util.spec_from_loader("browse", loader)
+browse = importlib.util.module_from_spec(spec)
+loader.exec_module(browse)
+
+
+if __name__ == "__main__":
+ browse.db_con = utils.init_db(":memory:", check_same_thread=False)
+ with browse.db_con:
+ tests.populate_dummy_db(browse.db_con)
+ browse.swift_container_url = "swift-%s"
+ browse.AMQP_QUEUE_CACHE = Path("/dev/shm/queue.json")
+ tests.populate_dummy_amqp_cache(browse.AMQP_QUEUE_CACHE)
+ browse.RUNNING_CACHE = Path("/dev/shm/running.json")
+ tests.populate_dummy_running_cache(browse.RUNNING_CACHE)
+
+ browse.app.run(host="0.0.0.0", debug=True)
diff --git a/charms/focal/autopkgtest-web/webcontrol/browse.cgi b/charms/focal/autopkgtest-web/webcontrol/browse.cgi
index f39b30f..f6be794 100755
--- a/charms/focal/autopkgtest-web/webcontrol/browse.cgi
+++ b/charms/focal/autopkgtest-web/webcontrol/browse.cgi
@@ -10,9 +10,9 @@ import sqlite3
from collections import OrderedDict
from wsgiref.handlers import CGIHandler
-import distro_info
import flask
from helpers.admin import select_abnormally_long_jobs
+from helpers.utils import get_all_releases, get_supported_releases
from werkzeug.middleware.proxy_fix import ProxyFix
app = flask.Flask("browse")
@@ -20,13 +20,11 @@ app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1)
db_con = None
swift_container_url = None
-UDI = distro_info.UbuntuDistroInfo()
-ALL_UBUNTU_RELEASES = UDI.all
-SUPPORTED_UBUNTU_RELEASES = sorted(
- set(UDI.supported() + UDI.supported_esm()), key=ALL_UBUNTU_RELEASES.index
-)
-
+ALL_UBUNTU_RELEASES = get_all_releases()
+SUPPORTED_UBUNTU_RELEASES = get_supported_releases()
INDEXED_PACKAGES_FP = ""
+AMQP_QUEUE_CACHE = "/var/lib/cache-amqp/queued.json"
+RUNNING_CACHE = "/run/amqp-status-collector/running.json"
def init_config():
@@ -60,6 +58,16 @@ def get_test_id(release, arch, src):
return None
+def get_running_jobs():
+ try:
+ with open(RUNNING_CACHE) as f:
+ # package -> runhash -> release -> arch -> (params, duration, logtail)
+ running_info = json.load(f)
+ except FileNotFoundError:
+ running_info = {}
+ return running_info
+
+
def render(template, code=200, **kwargs):
# sort the values passed in, so that releases are in the right order
try:
@@ -81,7 +89,7 @@ def render(template, code=200, **kwargs):
flask.render_template(
template,
base_url=flask.url_for("index_root"),
- static_url=flask.url_for("static", filename="/"),
+ static_url=flask.url_for("static", filename=""),
**kwargs
),
code,
@@ -147,7 +155,7 @@ def get_queue_info():
Return (releases, arches, context -> release -> arch -> (queue_size, [requests])).
"""
- with open("/var/lib/cache-amqp/queued.json", "r") as json_file:
+ with open(AMQP_QUEUE_CACHE, "r") as json_file:
queue_info_j = json.load(json_file)
arches = queue_info_j["arches"]
@@ -268,6 +276,10 @@ def package_overview(package, _=None):
arches.add(row[3])
results.setdefault(row[2], {})[row[3]] = human_exitcode(row[1])
+ running_info = dict(
+ (k, v) for (k, v) in get_running_jobs().items() if k == package
+ )
+
return render(
"browse-package.html",
package=package,
@@ -280,6 +292,7 @@ def package_overview(package, _=None):
arches=sorted(arches),
results=results,
title_suffix="- %s" % package,
+ running=running_info,
)
@@ -371,12 +384,7 @@ def running():
a
] = queue_length
- try:
- with open("/run/amqp-status-collector/running.json") as f:
- # package -> runhash -> release -> arch -> (params, duration, logtail)
- running_info = json.load(f)
- except FileNotFoundError:
- running_info = {}
+ running_info = get_running_jobs()
return render(
"browse-running.html",
@@ -391,13 +399,7 @@ def running():
@app.route("/admin")
def admin():
- try:
- with open("/run/amqp-status-collector/running.json") as f:
- # package -> runhash -> release -> arch -> (params, duration, logtail)
- running_info = json.load(f)
- except FileNotFoundError as exc:
- running_info = {}
- raise FileNotFoundError("running.json doesn't exist!") from exc
+ running_info = get_running_jobs()
pruned_running_info = select_abnormally_long_jobs(
running_info, get_test_id=get_test_id, db_con=db_con
)
@@ -439,7 +441,7 @@ def queues_json():
@app.route("/queued.json")
def return_queued_exactly():
- with open("/var/lib/cache-amqp/queued.json") as json_file:
+ with open(AMQP_QUEUE_CACHE) as json_file:
queue_info = json.load(json_file)
return queue_info
diff --git a/charms/focal/autopkgtest-web/webcontrol/helpers/tests.py b/charms/focal/autopkgtest-web/webcontrol/helpers/tests.py
new file mode 100644
index 0000000..52017c2
--- /dev/null
+++ b/charms/focal/autopkgtest-web/webcontrol/helpers/tests.py
@@ -0,0 +1,148 @@
+import json
+from datetime import datetime
+from uuid import uuid4
+
+from .utils import get_supported_releases
+
+
+def populate_dummy_db(db_con):
+ supported_releases = get_supported_releases()
+
+ c = db_con.cursor()
+ tests = [
+ (1, supported_releases[0], "amd64", "hello"),
+ (2, supported_releases[1], "amd64", "hello"),
+ (3, supported_releases[0], "ppc64el", "hello"),
+ (4, supported_releases[1], "ppc64el", "hello"),
+ (5, supported_releases[2], "amd64", "hello"),
+ ]
+ c.executemany("INSERT INTO test values(?, ?, ?, ?)", tests)
+ results = [
+ # fmt: off
+ # test_id | run_id | version | trigger | duration | exit_code | requester | env | uuid
+ (1, datetime.now(), "1.2.3", "hello/1.2.3", 42, 0, "hyask", "", str(uuid4())),
+ (1, datetime.now(), "1.2.3", "hello/1.2.3", 42, 0, "hyask", "all-proposed=1", str(uuid4())),
+ (2, datetime.now(), "1.2.3", "hello/1.2.3", 42, 0, "", "", str(uuid4())),
+ (3, datetime.now(), "1.2.3", "hello/1.2.3", 42, 20, "", "", str(uuid4())),
+ # fmt: on
+ ]
+ c.executemany(
+ "INSERT INTO result values(?, ?, ?, ?, ?, ?, ?, ?, ?)", results
+ )
+ db_con.commit()
+
+
+def populate_dummy_amqp_cache(path):
+ supported_releases = get_supported_releases()
+ with open(path, "w") as f:
+ # pylint: disable=line-too-long
+ json.dump(
+ {
+ "arches": ["amd64", "ppc64el"],
+ "queues": {
+ "ubuntu": {
+ supported_releases[0]: {
+ "amd64": {
+ "size": 1,
+ "requests": [
+ 'hello\n{"triggers": ["hello/1.2.3ubuntu1"], "submit-time": "2024-02-22 01:55:03"}',
+ ],
+ }
+ }
+ },
+ "huge": {
+ supported_releases[1]: {
+ "amd64": {
+ "size": 1,
+ "requests": [
+ 'hello\n{"triggers": ["migration-reference/0"], "submit-time": "2024-02-22 01:55:03"}',
+ ],
+ }
+ }
+ },
+ "ppa": {
+ supported_releases[2]: {
+ "amd64": {
+ "size": 2,
+ "requests": [
+ 'hello\n{"triggers": ["hello/1.2.4~ppa1"], "submit-time": "2024-02-22 01:55:03"}',
+ 'hello2\n{"triggers": ["hello2/2.0.0~ppa1"], "submit-time": "2024-02-22 01:55:03"}',
+ ],
+ }
+ }
+ },
+ "upstream": {
+ supported_releases[3]: {
+ "amd64": {
+ "size": 1,
+ "requests": [
+ 'hello\n{"triggers": ["hello/1.2.4~ppa1"], "submit-time": "2024-02-22 01:55:03"}',
+ ],
+ }
+ }
+ },
+ },
+ },
+ f,
+ )
+
+
+def populate_dummy_running_cache(path):
+ supported_releases = get_supported_releases()
+ with open(path, "w") as f:
+ json.dump(
+ {
+ "hello": {
+ "hash1": {
+ supported_releases[0]: {
+ "amd64": [
+ {
+ "requester": "hyask",
+ "submit-time": "2024-02-21 11:00:51",
+ "triggers": [
+ "hello/1.2.3",
+ ],
+ "uuid": "84669a9c-ac08-46a3-a5fd-6247d0d2021c",
+ },
+ 3504,
+ """
+3071s hello/test_XYZ.hello . [ 54%]
+3153s hello/test_XYZ.hello ...... [ 64%]
+3271s hello/test_XYZ.hello .......... [ 74%]
+3292s hello/test_XYZ.hello .................. [ 84%]
+3493s hello/test_XYZ.hello ............................ [ 94%]
+3494s hello/test_XYZ.hello .................................... [ 98%]
+""",
+ ]
+ }
+ }
+ },
+ "hello2": {
+ "hash1": {
+ supported_releases[4]: {
+ "amd64": [
+ {
+ "all-proposed": "1",
+ "requester": "hyask",
+ "submit-time": "2024-02-21 11:01:21",
+ "triggers": [
+ "hello2/1.2.3-0ubuntu1",
+ ],
+ "uuid": "42369a9c-ac08-46a3-a5fd-6247d0d2021c",
+ },
+ 3504,
+ """
+3071s hello2/test_XYZ.hello [ 54%]
+3153s hello2/test_XYZ.hello [ 64%]
+3271s hello2/test_XYZ.hello [ 74%]
+3292s hello2/test_XYZ.hello [ 84%]
+3493s hello2/test_XYZ.hello [ 94%]
+3494s hello2/test_XYZ.hello [ 98%]
+""",
+ ]
+ }
+ }
+ },
+ },
+ f,
+ )
diff --git a/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py b/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
index 58a9514..12d93b5 100644
--- a/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
+++ b/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
@@ -8,6 +8,22 @@ import random
import sqlite3
import time
+import distro_info
+
+
+def get_all_releases():
+ udi = distro_info.UbuntuDistroInfo()
+ return udi.all
+
+
+def get_supported_releases():
+ udi = distro_info.UbuntuDistroInfo()
+ all_ubuntu_releases = get_all_releases()
+ return sorted(
+ set(udi.supported() + udi.supported_esm()),
+ key=all_ubuntu_releases.index,
+ )
+
def setup_key(app, path):
"""Create or load app.secret_key for cookie encryption."""
@@ -22,10 +38,10 @@ def setup_key(app, path):
app.secret_key = key
-def init_db(path):
+def init_db(path, **kwargs):
"""Create DB if it does not exist, and connect to it"""
- db = sqlite3.connect(path)
+ db = sqlite3.connect(path, **kwargs)
c = db.cursor()
try:
c.execute("PRAGMA journal_mode = WAL")
diff --git a/charms/focal/autopkgtest-web/webcontrol/static/bootstrap b/charms/focal/autopkgtest-web/webcontrol/static/bootstrap
new file mode 120000
index 0000000..fe0f86b
--- /dev/null
+++ b/charms/focal/autopkgtest-web/webcontrol/static/bootstrap
@@ -0,0 +1 @@
+/usr/share/javascript/bootstrap
\ No newline at end of file
diff --git a/charms/focal/autopkgtest-web/webcontrol/templates/browse-admin.html b/charms/focal/autopkgtest-web/webcontrol/templates/browse-admin.html
index 5bc2459..266f584 100644
--- a/charms/focal/autopkgtest-web/webcontrol/templates/browse-admin.html
+++ b/charms/focal/autopkgtest-web/webcontrol/templates/browse-admin.html
@@ -1,29 +1,14 @@
{% extends "browse-layout.html" %}
+{% import "macros.html" as macros %}
+
{% block content %}
<h1 class="page-header">Admin</h1>
<p>Click on the package name to jump to the tests of the package for all arches/releases.</p>
<p>This page is simply a bunch of heuristics filtering all running jobs to try to get the problematic ones. Feel free to come help improve the heuristics <a href="https://code.launchpad.net/~ubuntu-release/autopkgtest-cloud/+git/autopkgtest-cloud/+ref/master";>here.</a></p>
<!-- Running tests -->
- {% for p in running|sort %}
- <h2 id="pkg-{{p}}"><a href="/packages/{{p}}">{{p}}</a></h2>
- {% for runhash, relinfo in running[p].items() %}
- {% for release, archinfo in relinfo.items() %}
- {% for arch, (params, duration, logtail) in archinfo.items() %}
- <table class="table-condensed">
- <tr><th>Release:</th><td>{{release}}</td></tr>
- <tr><th>Architecture:</th><td>{{arch}}</td></tr>
- {% for param, v in params.items() %}
- <tr><th>{{param|capitalize}}:</th><td>{{v}}</td></tr>
- {% endfor %}
- <tr><th>Running for:</th><td>{{duration//3600 }}h {{duration % 3600//60}}m {{duration % 60}}s</td></tr>
- </table>
- <pre>
-{{logtail}}
- </pre>
- {% endfor %}
- {% endfor %}
- {% endfor %}
+ {% for p, info in running.items()|sort %}
+ {{ macros.display_running_job(p, info) }}
{% endfor %}
{% endblock %}
diff --git a/charms/focal/autopkgtest-web/webcontrol/templates/browse-layout.html b/charms/focal/autopkgtest-web/webcontrol/templates/browse-layout.html
index 0f90de3..c91457d 100644
--- a/charms/focal/autopkgtest-web/webcontrol/templates/browse-layout.html
+++ b/charms/focal/autopkgtest-web/webcontrol/templates/browse-layout.html
@@ -6,9 +6,9 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Ubuntu Autopkgtest Results {{title_suffix}}</title>
<!-- <link rel="icon" type="image/png" href="/debian.png"/> -->
- <link rel="stylesheet" type="text/css" href="{{static_url}}/bootstrap/css/bootstrap.css"/>
- <link rel="stylesheet" type="text/css" href="{{static_url}}/bootstrap/css/bootstrap-theme.css"/>
- <link rel="stylesheet" type="text/css" href="{{static_url}}/style.css"/>
+ <link rel="stylesheet" type="text/css" href="{{static_url}}bootstrap/css/bootstrap.css"/>
+ <link rel="stylesheet" type="text/css" href="{{static_url}}bootstrap/css/bootstrap-theme.css"/>
+ <link rel="stylesheet" type="text/css" href="{{static_url}}style.css"/>
</head>
<body>
<div id='wrap'>
@@ -44,7 +44,7 @@
{% block content %}{% endblock %}
</div>
- <script type="text/javascript" src="{{static_url}}/jquery/jquery.min.js"></script>
- <script type="text/javascript" src="{{static_url}}/bootstrap/js/bootstrap.min.js"></script>
+ <script type="text/javascript" src="{{static_url}}jquery/jquery.min.js"></script>
+ <script type="text/javascript" src="{{static_url}}bootstrap/js/bootstrap.min.js"></script>
</body>
</html>
diff --git a/charms/focal/autopkgtest-web/webcontrol/templates/browse-package.html b/charms/focal/autopkgtest-web/webcontrol/templates/browse-package.html
index 3ac81c3..fb12afa 100644
--- a/charms/focal/autopkgtest-web/webcontrol/templates/browse-package.html
+++ b/charms/focal/autopkgtest-web/webcontrol/templates/browse-package.html
@@ -1,4 +1,6 @@
{% extends "browse-layout.html" %}
+{% import "macros.html" as macros %}
+
{% block content %}
<h2>{{package}}</h2>
@@ -17,4 +19,8 @@
</tr>
{% endfor %}
</table>
+
+ {% for p, info in running.items()|sort %}
+ {{ macros.display_running_job(p, info) }}
+ {% endfor %}
{% endblock %}
diff --git a/charms/focal/autopkgtest-web/webcontrol/templates/browse-running.html b/charms/focal/autopkgtest-web/webcontrol/templates/browse-running.html
index 3711b97..53ae2f9 100644
--- a/charms/focal/autopkgtest-web/webcontrol/templates/browse-running.html
+++ b/charms/focal/autopkgtest-web/webcontrol/templates/browse-running.html
@@ -1,4 +1,6 @@
{% extends "browse-layout.html" %}
+{% import "macros.html" as macros %}
+
{% block content %}
<h1 class="page-header">Currently running tests</h1>
<p>Click on the package name to jump to the currently running tests of that package.</p>
@@ -36,31 +38,8 @@
{% endfor %}
<!-- Running tests -->
- {% for p in running|sort %}
- <h2 id="pkg-{{p}}"><a href="/packages/{{p}}">{{p}}</a></h2>
- {% for runhash, relinfo in running[p].items() %}
- {% for release, archinfo in relinfo.items() %}
- {% for arch, (params, duration, logtail) in archinfo.items() %}
- <table class="table-condensed">
- <tr><th>Release:</th><td>{{release}}</td></tr>
- <tr><th>Architecture:</th><td>{{arch}}</td></tr>
- {% for param, v in params.items() %}
- {% if param == "requester" %}
- <tr><th>{{param|capitalize}}:</th><td><a href="https://launchpad.net/~{{v}}";>{{v}}</a></td></tr>
- {% elif param == "uuid" %}
- <tr><th>{{param|capitalize}}:</th><td>{{v}}</td></tr>
- {% else %}
- <tr><th>{{param|capitalize}}:</th><td>{{v}}</td></tr>
- {% endif %}
- {% endfor %}
- <tr><th>Running for:</th><td>{{duration//3600 }}h {{duration % 3600//60}}m {{duration % 60}}s</td></tr>
- </table>
- <pre>
-{{logtail}}
- </pre>
- {% endfor %}
- {% endfor %}
- {% endfor %}
+ {% for p, info in running.items()|sort %}
+ {{ macros.display_running_job(p, info) }}
{% endfor %}
<!-- queue contents -->
diff --git a/charms/focal/autopkgtest-web/webcontrol/templates/macros.html b/charms/focal/autopkgtest-web/webcontrol/templates/macros.html
new file mode 100644
index 0000000..7d43d20
--- /dev/null
+++ b/charms/focal/autopkgtest-web/webcontrol/templates/macros.html
@@ -0,0 +1,26 @@
+{% macro display_running_job(package, info) -%}
+<h2 id="pkg-{{ package }}"><a href="/packages/{{ package }}">{{ package }}</a></h2>
+ {% for runhash, relinfo in info.items() %}
+ {% for release, archinfo in relinfo.items() %}
+ {% for arch, (params, duration, logtail) in archinfo.items() %}
+ <table class="table-condensed">
+ <tr><th>Release:</th><td>{{ release }}</td></tr>
+ <tr><th>Architecture:</th><td>{{ arch }}</td></tr>
+ {% for param, v in params.items() %}
+ {% if param == "requester" %}
+ <tr><th>{{ param|capitalize }}:</th><td><a href="https://launchpad.net/~{{ v }}">{{ v }}</a></td></tr>
+ {% elif param == "uuid" %}
+ <tr><th>{{ param|upper }}:</th><td>{{ v }}</td></tr>
+ {% else %}
+ <tr><th>{{ param|capitalize }}:</th><td>{{ v }}</td></tr>
+ {% endif %}
+ {% endfor %}
+ <tr><th>Running for:</th><td>{{ duration//3600 }}h {{ duration % 3600//60 }}m {{ duration % 60 }}s ({{ duration }}s)</td></tr>
+ </table>
+ <pre>
+{{ logtail }}
+ </pre>
+ {% endfor %}
+ {% endfor %}
+ {% endfor %}
+{%- endmacro %}
Follow ups
