canonical-ubuntu-qa team mailing list archive

[Ural Tunaboyu <mp+477471@xxxxxxxxxxxxxxxxxx>]

Ural Tunaboyu has proposed merging ~uralt/autopkgtest-cloud:optimize-testlist into autopkgtest-cloud:master.

Requested reviews:
  Canonical's Ubuntu QA (canonical-ubuntu-qa)

For more details, see:
https://code.launchpad.net/~uralt/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/477471

Several small changes to the /testlist endpoint which I noticed were needed when browsing:
 - moved duplicate code for generating indexed-packages list to utils.py
 - omit "unknown" as a valid version when getting latest package version
 - made sure to write generated indexed-packages list to disk to avoid duplicate computation
 - improved readability of testlist page by adding space between table and index
-- 
Your team Canonical's Ubuntu QA is requested to review the proposed merge of ~uralt/autopkgtest-cloud:optimize-testlist into autopkgtest-cloud:master.

diff --git a/charms/focal/autopkgtest-web/webcontrol/browse.cgi b/charms/focal/autopkgtest-web/webcontrol/browse.cgi
index 3e7838a..5459605 100755
--- a/charms/focal/autopkgtest-web/webcontrol/browse.cgi
+++ b/charms/focal/autopkgtest-web/webcontrol/browse.cgi
@@ -26,9 +26,11 @@ from helpers.utils import (
     get_all_releases,
     get_autopkgtest_cloud_conf,
     get_fallback_stats_dir,
+    get_indexed_packages,
     get_release_arches,
     get_supported_releases,
     setup_key,
+    srchash,
 )
 from werkzeug.middleware.proxy_fix import ProxyFix
 
@@ -173,13 +175,6 @@ def human_exitcode(code):
     return "exit code %i" % code
 
 
-def srchash(src):
-    if src.startswith("lib"):
-        return src[:4]
-    else:
-        return src[0]
-
-
 def get_queues_info():
     """Return information about queued tests
 
@@ -893,17 +888,9 @@ def testlist():
         with open(CONFIG["indexed_packages"], "r") as f:
             indexed_pkgs = json.load(f)
     except FileNotFoundError:
-        indexed_pkgs = {}
-        for row in db_con.execute(
-            "SELECT package, MAX(version) "
-            "FROM test, result "
-            "WHERE id == test_id "
-            "GROUP BY package "
-            "ORDER BY package"
-        ):
-            # strip off epoch
-            v = row[1][row[1].find(":") + 1 :]
-            indexed_pkgs.setdefault(srchash(row[0]), []).append((row[0], v))
+        indexed_pkgs = get_indexed_packages()
+        with open(CONFIG["indexed_packages"], "w") as f:
+            json.dump(indexed_pkgs, f)
     return render("browse-testlist.html", indexed_pkgs=indexed_pkgs)
 
 
diff --git a/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py b/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
index 5c3e4ce..a801217 100644
--- a/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
+++ b/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
@@ -209,6 +209,42 @@ def get_github_context(params: typing.Dict[str, str]) -> str:
         return "%s/%s" % (params["release"], params["arch"])
 
 
+def srchash(src: str) -> str:
+    """Get srchash of package name
+
+    :param src:
+        package name
+    :type src: ``str``
+    :return ``str``:
+        The first letter of package name, or if
+        package name starts with 'lib' then
+        'lib' + first letter
+    """
+    if src.startswith("lib"):
+        return src[:4]
+    else:
+        return src[0]
+
+
+def get_indexed_packages():
+    db_con = get_autopkgtest_db_conn()
+    indexed_packages = {}
+
+    for row in db_con.execute(
+        "SELECT package, MAX(version) "
+        "FROM test, result "
+        "WHERE id == test_id "
+        "AND version != 'unknown' "
+        "GROUP BY package "
+        "ORDER BY package"
+    ):
+        # strip off epoch
+        v = row[1][row[1].find(":") + 1 :]
+        indexed_packages.setdefault(srchash(row[0]), []).append((row[0], v))
+
+    return indexed_packages
+
+
 class timeout:
     def __init__(self, seconds=1, error_message="Timeout"):
         self.seconds = seconds
diff --git a/charms/focal/autopkgtest-web/webcontrol/indexed-packages b/charms/focal/autopkgtest-web/webcontrol/indexed-packages
index 114d6d8..c032a5f 100755
--- a/charms/focal/autopkgtest-web/webcontrol/indexed-packages
+++ b/charms/focal/autopkgtest-web/webcontrol/indexed-packages
@@ -1,37 +1,12 @@
 #!/usr/bin/python3
-
 import json
-import sqlite3
-
-from helpers.utils import get_autopkgtest_cloud_conf
-
-
-def srchash(src):
-    if src.startswith("lib"):
-        return src[:4]
-    else:
-        return src[0]
 
+from helpers.utils import get_autopkgtest_cloud_conf, get_indexed_packages
 
 if __name__ == "__main__":
     cp = get_autopkgtest_cloud_conf()
-    indexed_packages = cp["web"]["indexed_packages"]
+    indexed_packages_fp = cp["web"]["indexed_packages"]
 
-    db_con = sqlite3.connect(
-        "file:%s?mode=ro" % cp["web"]["database_ro"],
-        uri=True,
-        check_same_thread=False,
-    )
-    indexed_packages = {}
-    for row in db_con.execute(
-        "SELECT package, MAX(version) "
-        "FROM test, result "
-        "WHERE id == test_id "
-        "GROUP BY package "
-        "ORDER BY package"
-    ):
-        # strip off epoch
-        v = row[1][row[1].find(":") + 1 :]
-        indexed_packages.setdefault(srchash(row[0]), []).append((row[0], v))
-    with open(indexed_packages, "w") as f:
+    indexed_packages = get_indexed_packages()
+    with open(indexed_packages_fp, "w") as f:
         json.dump(indexed_packages, f)
diff --git a/charms/focal/autopkgtest-web/webcontrol/templates/browse-testlist.html b/charms/focal/autopkgtest-web/webcontrol/templates/browse-testlist.html
index 73ac649..7309935 100644
--- a/charms/focal/autopkgtest-web/webcontrol/templates/browse-testlist.html
+++ b/charms/focal/autopkgtest-web/webcontrol/templates/browse-testlist.html
@@ -41,7 +41,7 @@
           {% endfor %}
         </table>
       </div>
-
+      <div class="col-md-2"></div>
       <div class="col-md-3">
         <div class="panel panel-default">
           <div class="panel-heading">Package Index</div>