canonical-ubuntu-qa team mailing list archive

[Tim Andersson <mp+463146@xxxxxxxxxxxxxxxxxx>]

Tim Andersson has proposed merging ~andersson123/autopkgtest-cloud:auto-queue-cleanup into autopkgtest-cloud:master.

Requested reviews:
  Canonical's Ubuntu QA (canonical-ubuntu-qa)

For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/463146
-- 
Your team Canonical's Ubuntu QA is requested to review the proposed merge of ~andersson123/autopkgtest-cloud:auto-queue-cleanup into autopkgtest-cloud:master.

diff --git a/charms/focal/autopkgtest-web/webcontrol/queue-merger b/charms/focal/autopkgtest-web/webcontrol/queue-merger
new file mode 100755
index 0000000..f1f9f81
--- /dev/null
+++ b/charms/focal/autopkgtest-web/webcontrol/queue-merger
@@ -0,0 +1,111 @@
+#!/usr/bin/python3
+
+# import argparse
+import apt_pkg
+import json
+import itertools
+import os
+import apt.progress.base
+# import urllib
+
+# steal from here:
+# https://gist.github.com/julian-klode/2785b07e76abaed78d34242a39d9bf0b
+
+
+"""
+I think maybe this has to run on the cloud worker to run filter-amqp? Needs some more thought.
+
+contexts = ["huge", "ppa", "upstream", "ubuntu"]
+
+Steps:
+for release in releases:
+    for arch in arches:
+        for queue in queue_names:
+            get queue
+            get proposed packages
+            calculate merged queue
+            empty context/release/arch queue
+            re-queue the merged queue
+
+make sure to differentiate between huge and non-huge queues
+remember to add logging!!!
+"""
+
+def get_queue(release, arch, context, all_queues):
+    """Build a single queue of package, args tuples."""
+    queue = [
+        (i.split()[0], json.loads(i.split(None, 1)[1]))
+        for i in all_queues["queues"][context][release][arch]["requests"]
+    ]
+    return queue
+
+
+def get_proposed_packages():
+    """Get all valid triggers in proposed, returns set of package/version strings."""
+    tmpdir = "merge-requests.tmp"
+    os.makedirs(os.path.join(tmpdir, "etc/apt"), exist_ok=True)
+    os.makedirs(os.path.join(tmpdir, "var/lib/apt"), exist_ok=True)
+    with open(os.path.join(tmpdir, "etc/apt/sources.list"), "w") as sources:
+        print(
+            "deb-src [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] https://snapshot.ubuntu.com/ubuntu noble-proposed main universe restricted multiverse",
+            file=sources,
+        )
+
+    apt_pkg.config["Dir"] = tmpdir
+    apt_pkg.init()
+    sl = apt_pkg.SourceList()
+    sl.read_main_list()
+    cache = apt_pkg.Cache(None)
+    cache.update(apt.progress.base.AcquireProgress(), sl)
+
+    srcrecords = apt_pkg.SourceRecords()
+
+    proposed = set()
+    while srcrecords.step():
+        proposed.add(srcrecords.package + "/" + srcrecords.version)
+    return proposed
+
+
+def merge_queue(queue, proposed):
+    """Calculate the merged queue"""
+    out = []
+    for package, runs in itertools.groupby(
+        sorted(queue, key=lambda i: i[0]), lambda i: i[0]
+    ):
+        triggers = set()
+        for package, args in runs:
+            for trigger in args["triggers"]:
+                if trigger in proposed or trigger == "migration-reference/0":
+                    triggers.add(trigger)
+                else:
+                    print(
+                        "Dropping obsolete trigger",
+                        trigger,
+                        "for",
+                        package,
+                        file=sys.stderr,
+                    )
+
+        if "migration-reference/0" in triggers:
+            out.append((package, {"triggers": ["migration-reference/0"]}))
+            triggers.remove("migration-reference/0")
+        if triggers:
+            out.append((package, {"all-proposed": "1", "triggers": sorted(triggers)}))
+    return out
+
+
+def main():
+    with open("/var/lib/cache-amqp/queued.json", "r") as f:
+        all_queue_info  = json.load(f)
+    contexts = ["huge", "ppa", "upstream", "ubuntu"]
+    arches = all_queue_info.get("arches")
+    releases = all_queue_info.get("releases")
+    proposed_packages = get_proposed_packages()
+    for context in contexts:
+        for release in releases:
+            for arch in arches:
+                queue_name = f"debci-{context}-{release}-{arch}" if context != "ubuntu" else f"debci-{release}-{arch}"
+                individual_queue = get_queue(release, arch, context, all_queue_info)
+                merged_queue = merge_queue(individual_queue, proposed_packages)
+                for pkg, run_args in merged_queue:
+                    print(run_args)