canonical-ubuntu-qa team mailing list archive

[Tim Andersson <mp+458001@xxxxxxxxxxxxxxxxxx>]

Tim Andersson has proposed merging ~andersson123/autopkgtest-cloud:improve-loading-results into autopkgtest-cloud:master.

Requested reviews:
  Canonical's Ubuntu QA (canonical-ubuntu-qa)

For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/458001
-- 
Your team Canonical's Ubuntu QA is requested to review the proposed merge of ~andersson123/autopkgtest-cloud:improve-loading-results into autopkgtest-cloud:master.

diff --git a/charms/focal/autopkgtest-web/units/indexed-packages.service b/charms/focal/autopkgtest-web/units/indexed-packages.service
new file mode 100644
index 0000000..e518740
--- /dev/null
+++ b/charms/focal/autopkgtest-web/units/indexed-packages.service
@@ -0,0 +1,6 @@
+[Unit]
+Description=Get index of packages
+
+[Service]
+Type=oneshot
+ExecStart=/home/ubuntu/webcontrol/indexed-packages
diff --git a/charms/focal/autopkgtest-web/units/indexed-packages.timer b/charms/focal/autopkgtest-web/units/indexed-packages.timer
new file mode 100644
index 0000000..35066d3
--- /dev/null
+++ b/charms/focal/autopkgtest-web/units/indexed-packages.timer
@@ -0,0 +1,9 @@
+[Unit]
+Description=Get index of packages
+
+[Timer]
+OnCalendar=*:0/5
+
+
+[Install]
+WantedBy=autopkgtest-web.target
diff --git a/charms/focal/autopkgtest-web/webcontrol/browse.cgi b/charms/focal/autopkgtest-web/webcontrol/browse.cgi
index f2dc00f..eaf6915 100755
--- a/charms/focal/autopkgtest-web/webcontrol/browse.cgi
+++ b/charms/focal/autopkgtest-web/webcontrol/browse.cgi
@@ -25,6 +25,17 @@ SUPPORTED_UBUNTU_RELEASES = sorted(
     set(UDI.supported() + UDI.supported_esm()), key=ALL_UBUNTU_RELEASES.index
 )
 
+# INDEXED_PACKAGES = {}
+# LOCK = Lock()
+
+# INDEXED_PACKAGES_FILEPATH = (
+#     "/var/www/indexed-packages-"
+#     + "".join(random.choice(string.ascii_lowercase) for _ in range(5))
+#     + ".json"
+# )
+
+INDEXED_PACKAGES_FP = "/home/ubuntu/indexed-packages.json"
+
 
 def init_config():
     global db_con, swift_container_url
@@ -33,7 +44,9 @@ def init_config():
     cp.read(os.path.expanduser("~ubuntu/autopkgtest-cloud.conf"))
 
     db_con = sqlite3.connect(
-        "file:%s?mode=ro" % cp["web"]["database_ro"], uri=True
+        "file:%s?mode=ro" % cp["web"]["database_ro"],
+        uri=True,
+        check_same_thread=False,
     )
     try:
         url = cp["web"]["ExternalURL"]
@@ -409,19 +422,13 @@ def return_queued_exactly():
 
 @app.route("/testlist")
 def testlist():
-    # geneate index → [(package, version)] map
-    indexed_pkgs = {}
-    for row in db_con.execute(
-        "SELECT package, MAX(version) "
-        "FROM test, result "
-        "WHERE id == test_id "
-        "GROUP BY package "
-        "ORDER BY package"
-    ):
-        # strip off epoch
-        v = row[1][row[1].find(":") + 1 :]
-        indexed_pkgs.setdefault(srchash(row[0]), []).append((row[0], v))
-    return render("browse-testlist.html", indexed_pkgs=indexed_pkgs)
+    try:
+        indexed_packages = {}
+        with open(INDEXED_PACKAGES_FP, "r") as f:
+            indexed_packages = json.load(f)
+    except FileNotFoundError:
+        return None
+    return render("browse-testlist.html", indexed_pkgs=indexed_packages)
 
 
 @app.route("/statistics")
@@ -480,5 +487,8 @@ def statistics():
 
 if __name__ == "__main__":
     app.config["DEBUG"] = True
+    # thread = Thread(target=testlist_in_background)
+    # thread.daemon = True
     init_config()
+    # thread.start()
     CGIHandler().run(app)
diff --git a/charms/focal/autopkgtest-web/webcontrol/indexed-packages b/charms/focal/autopkgtest-web/webcontrol/indexed-packages
new file mode 100755
index 0000000..383ffbc
--- /dev/null
+++ b/charms/focal/autopkgtest-web/webcontrol/indexed-packages
@@ -0,0 +1,39 @@
+#!/usr/bin/python3
+
+import configparser
+import json
+import os
+import sqlite3
+
+INDEXED_PACKAGES_FP = "/home/ubuntu/indexed-packages.json"
+
+
+def srchash(src):
+    if src.startswith("lib"):
+        return src[:4]
+    else:
+        return src[0]
+
+
+if __name__ == "__main__":
+    cp = configparser.ConfigParser()
+    cp.read(os.path.expanduser("~ubuntu/autopkgtest-cloud.conf"))
+
+    db_con = sqlite3.connect(
+        "file:%s?mode=ro" % cp["web"]["database_ro"],
+        uri=True,
+        check_same_thread=False,
+    )
+    indexed_packages = {}
+    for row in db_con.execute(
+        "SELECT package, MAX(version) "
+        "FROM test, result "
+        "WHERE id == test_id "
+        "GROUP BY package "
+        "ORDER BY package"
+    ):
+        # strip off epoch
+        v = row[1][row[1].find(":") + 1 :]
+        indexed_packages.setdefault(srchash(row[0]), []).append((row[0], v))
+    with open(INDEXED_PACKAGES_FP, "w") as f:
+        json.dump(indexed_packages, f)