canonical-ubuntu-qa team mailing list archive

[Tim Andersson <mp+471633@xxxxxxxxxxxxxxxxxx>]

Tim Andersson has proposed merging ~andersson123/autopkgtest-cloud:d-a-r-why-keep-dying into autopkgtest-cloud:master.

Requested reviews:
  Canonical's Ubuntu QA (canonical-ubuntu-qa)

For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/471633

Fix download-all-results OOM'ing by iterating through each container in batches
-- 
Your team Canonical's Ubuntu QA is requested to review the proposed merge of ~andersson123/autopkgtest-cloud:d-a-r-why-keep-dying into autopkgtest-cloud:master.

diff --git a/charms/focal/autopkgtest-web/webcontrol/download-all-results b/charms/focal/autopkgtest-web/webcontrol/download-all-results
index 0798695..4d6a807 100755
--- a/charms/focal/autopkgtest-web/webcontrol/download-all-results
+++ b/charms/focal/autopkgtest-web/webcontrol/download-all-results
@@ -53,10 +53,12 @@ def amqp_connect():
     return amqp_con
 
 
-def list_remote_container(container_name, swift_conn):
+def list_remote_container(container_name, swift_conn, marker, limit=1000):
     LOGGER.debug("Listing container %s", container_name)
     _, list_of_test_results = swift_conn.get_container(
-        container_name, full_listing=True
+        container_name,
+        marker=marker,
+        limit=limit,
     )
     ret_me = {}
     for result in list_of_test_results:
@@ -190,27 +192,35 @@ def fetch_one_result(container_name, object_name, swift_conn):
 def fetch_container(release, swift_conn):
     """Download new results from a swift container"""
     container_name = "autopkgtest-" + release
+    marker = None
 
-    try:
-        our_results = list_our_results(release)
-        known_results = list_remote_container(container_name, swift_conn)
+    while True:
+        try:
+            last_marker = marker
+            our_results = list_our_results(release)
+            known_results = list_remote_container(
+                container_name, swift_conn, marker
+            )
+            marker = known_results[list(known_results.keys())[-1]]
+            if last_marker == marker:
+                return
 
-        need_to_fetch = set(known_results.keys()) - our_results
+            need_to_fetch = set(known_results.keys()) - our_results
 
-        LOGGER.debug("Need to download %d items", len(need_to_fetch))
+            LOGGER.debug("Need to download %d items", len(need_to_fetch))
 
-        for run_id in need_to_fetch:
-            fetch_one_result(
-                container_name=container_name,
-                object_name=known_results[run_id],
-                swift_conn=swift_conn,
+            for run_id in need_to_fetch:
+                fetch_one_result(
+                    container_name=container_name,
+                    object_name=known_results[run_id],
+                    swift_conn=swift_conn,
+                )
+        except swiftclient.ClientException as e:
+            LOGGER.error(
+                "Something went wrong accessing container %s\nTraceback: %s"
+                % (container_name, str(e))
             )
-    except swiftclient.ClientException as e:
-        LOGGER.error(
-            "Something went wrong accessing container %s\nTraceback: %s"
-            % (container_name, str(e))
-        )
-        raise
+            raise
 
 
 if __name__ == "__main__":