canonical-ubuntu-qa team mailing list archive

[Tim Andersson <mp+445950@xxxxxxxxxxxxxxxxxx>]

Tim Andersson has proposed merging ~andersson123/autopkgtest-cloud:fix_seed_new_release into autopkgtest-cloud:master.

Commit message:
fix: seed new release retries in more appropriate manner

Requested reviews:
  Canonical's Ubuntu QA (canonical-ubuntu-qa)

For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/445950

fix: seed new release retries in more appropriate manner
-- 
Your team Canonical's Ubuntu QA is requested to review the proposed merge of ~andersson123/autopkgtest-cloud:fix_seed_new_release into autopkgtest-cloud:master.

diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/seed-new-release b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/seed-new-release
index 82547e1..d0233b0 100755
--- a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/seed-new-release
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/seed-new-release
@@ -30,10 +30,13 @@ def connect_swift():
         tenant_name=os.environ["OS_TENANT_NAME"],
         os_options={"region_name": os.environ["OS_REGION_NAME"]},
         auth_version="2.0",
+        retries=10,
+        starting_backoff=10,
+        force_auth_retry=True,
     )
 
 
-def copy_result(rel_path, source, target):
+def copy_result(rel_path, source, target, retries=0):
     global swift_con
 
     from_path = source.split("-")[-1] + rel_path
@@ -55,34 +58,38 @@ def copy_result(rel_path, source, target):
         except KeyError:
             pass
 
-    for retry in range(10):
-        try:
-            if not args.dry_run:
-                print("Putting %s" % from_path)
-                swiftclient.put_object(
-                    swift_con.url,
-                    token=swift_con.token,
-                    container=target,
-                    name=to_path,
-                    contents=contents,
-                    content_length=headers["content-length"],
-                    headers=headers_to_copy,
-                )
-            break
-        except (
-            IOError,
-            AttributeError,
-            swiftclient.exceptions.ClientException,
-        ) as e:
-            print(
-                "Error connecting to swift, re-connecting in %is: %s"
-                % (5 * retry, e)
+    # only trouble is there's no sys exit now ...
+    try:
+        if not args.dry_run:
+            print("Putting %s" % from_path)
+            # 'vanilla' retries now embedded in this
+            swiftclient.put_object(
+                swift_con.url,
+                token=swift_con.token,
+                container=target,
+                name=to_path,
+                contents=contents,
+                content_length=headers["content-length"],
+                headers=headers_to_copy,
             )
-            time.sleep(5 * retry)
-            swift_con = connect_swift()
-    else:
-        print("Repeated failure to connect to swift")
-        sys.exit(1)
+            print("Object %s put onto swift")
+        return
+    except (
+        IOError,
+        AttributeError,
+        swiftclient.exceptions.ClientException,
+    ) as e:
+        if retries == 10:
+            print("Retries exceeded, exiting.")
+            sys.exit(1)
+        print(
+            "Error connecting to swift, closing connection and trying again: \n %s",
+            e,
+        )  # will change
+        swift_con.close()
+        time.sleep(retries)
+        swift_con = connect_swift()
+        return copy_result(rel_path, source, target, retries + 1)
 
 
 def copy_run_results(