canonical-ubuntu-qa team mailing list archive

[Tim Andersson <mp+470103@xxxxxxxxxxxxxxxxxx>]

Tim Andersson has proposed merging ~andersson123/autopkgtest-cloud:s-n-r-prepend-series-version-to-uuid into autopkgtest-cloud:master.

Commit message:
fix: seed-new-release: modify uuid before copying testinfo.json

We realised recently that the uniqueness of the uuid column in the
database was causing issues when running seed-new-release. The summary
of the problem is as follows:

- seed-new-release copies swift objects from the old to new container
    - the two objects, in testinfo.json within result.tar have the
      same uuid
- we then run download-all-results for the new release
    - the way the results are entered into the db is with:
      INSERT OR REPLACE
    - the uuid column is unique, meaning the database entry for the old
      release gets REPLACE'd.
    - We longer have the result for the old release in the db
    - If one were to then run download-all-results for the old release,
      the results from the new release would then get removed from the
      database also.

This commit amends the issue by prepending the uuid in testinfo.json
with the series version of the old release before uploading it to the
container for the new release, preserving the database entries for both
the old and new releases.

Requested reviews:
  Canonical's Ubuntu QA (canonical-ubuntu-qa)

For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/470103

Fixes the issue with uuid unique-ness, seed-new-release and download-all-results by prepending the UUID with the old series version, like so:
2ff398c6-2fe0-4b74-a700-4c4cd54bd032 -> 2404-2ff398c6-2fe0-4b74-a700-4c4cd54bd032
-- 
Your team Canonical's Ubuntu QA is requested to review the proposed merge of ~andersson123/autopkgtest-cloud:s-n-r-prepend-series-version-to-uuid into autopkgtest-cloud:master.

diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/seed-new-release b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/seed-new-release
index ca047b6..b1bcd5b 100755
--- a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/seed-new-release
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/seed-new-release
@@ -7,12 +7,58 @@
 from __future__ import print_function
 
 import argparse
+import io
+import json
 import os
 import sqlite3
 import sys
+import tarfile
+import tempfile
 import time
 
 import swiftclient
+from distro_info import UbuntuDistroInfo
+
+
+def prepend_old_series_version_to_uuid(contents, old_series_version):
+    try:
+        object_tar_bytes = io.BytesIO(contents)
+        result_tar = {}
+        temp_dir = tempfile.TemporaryDirectory()
+        result_dir = temp_dir.name + "/result"
+        os.makedirs(result_dir)
+        testinfo = {}
+        with tarfile.open(None, "r", object_tar_bytes) as tar:
+            testinfo = json.loads(
+                tar.extractfile("testinfo.json").read().decode()
+            )
+            testinfo["uuid"] = f"{old_series_version}-{testinfo['uuid']}"
+            existing_files = tar.getnames()
+            for f in existing_files:
+                result_tar[f] = tar.extractfile(f).read().decode().strip()
+        result_tar["testinfo.json"] = testinfo
+        # write the files to new directory for tar'ing
+        for file_name, values in result_tar.items():
+            with open(f"{result_dir}/{file_name}", "w") as f:
+                if file_name.endswith(".json"):
+                    json.dump(values, f)
+                else:
+                    f.write(values)
+        # tar up the directory
+        output_result_tar = temp_dir.name + "/result.tar"
+        with tarfile.open(output_result_tar, "w") as tar:
+            for fn in os.listdir(result_dir):
+                p = os.path.join(result_dir, fn)
+                tar.add(p, arcname=fn)
+        new_content = None
+        with open(output_result_tar, "rb") as f:
+            new_content = f.read()
+        return new_content
+    except Exception as e:
+        print(f"Modifying uuid failed with {e} - not modifying")
+        return contents
+    finally:
+        temp_dir.cleanup()
 
 
 def srchash(src):
@@ -40,7 +86,7 @@ def connect_swift():
     )
 
 
-def copy_result(rel_path, source, target, retries=0):
+def copy_result(rel_path, source, target, old_series_version, retries=0):
     global swift_con
 
     from_path = source.split("-")[-1] + rel_path
@@ -62,6 +108,19 @@ def copy_result(rel_path, source, target, retries=0):
         except KeyError:
             pass
 
+    if "result.tar" in from_path:
+        old_contents = contents
+        contents = prepend_old_series_version_to_uuid(
+            contents, old_series_version
+        )
+        # if old_contents and contents are the same, it means
+        # prepend_old_series_version_to_uuid failed in some way.
+        # In this case, we don't want to continue this function,
+        # otherwise the object will be copied over with the same
+        # uuid.
+        if old_contents == contents:
+            return
+
     try:
         if not args.dry_run:
             print("Putting %s" % to_path)
@@ -92,12 +151,24 @@ def copy_result(rel_path, source, target, retries=0):
         swift_con.close()
         time.sleep(retries)
         swift_con = connect_swift()
-        return copy_result(rel_path, source, target, retries + 1)
+        return copy_result(
+            rel_path, source, target, old_series_version, retries + 1
+        )
 
 
 def copy_run_results(
-    package, arch, run_id, args, source_container, target_container
+    package,
+    arch,
+    run_id,
+    args,
+    source_container,
+    target_container,
+    old_release,
 ):
+    old_series_version = (
+        UbuntuDistroInfo().version(old_release).replace(" LTS", "")
+    )
+    old_series_version = "".join(old_series_version.split("."))
     for file in "artifacts.tar.gz", "result.tar", "log.gz":
         path = "/%s/%s/%s/%s/%s" % (
             arch,
@@ -109,7 +180,9 @@ def copy_run_results(
         if args.new_release + path in existing:
             print("%s%s already exists, skipping" % (args.old_release, path))
             continue
-        copy_result(path, source_container, target_container)
+        copy_result(
+            path, source_container, target_container, old_series_version
+        )
 
 
 ap = argparse.ArgumentParser()
@@ -169,7 +242,13 @@ for package, arch, run_id in db_con.execute(
     "GROUP BY package, arch" % args.old_release
 ):
     copy_run_results(
-        package, arch, run_id, args, source_container, target_container
+        package,
+        arch,
+        run_id,
+        args,
+        source_container,
+        target_container,
+        args.old_release,
     )
 
 for package, arch, run_id in db_con.execute(
@@ -180,5 +259,11 @@ for package, arch, run_id in db_con.execute(
     "GROUP BY package, arch" % args.old_release
 ):
     copy_run_results(
-        package, arch, run_id, args, source_container, target_container
+        package,
+        arch,
+        run_id,
+        args,
+        source_container,
+        target_container,
+        args.old_release,
     )