canonical-ubuntu-qa team mailing list archive

[Tim Andersson <mp+458950@xxxxxxxxxxxxxxxxxx>]

Tim Andersson has proposed merging ~andersson123/autopkgtest-cloud:uuid-integration into autopkgtest-cloud:master.

Requested reviews:
  Canonical's Ubuntu QA (canonical-ubuntu-qa)

For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/458950
-- 
Your team Canonical's Ubuntu QA is requested to review the proposed merge of ~andersson123/autopkgtest-cloud:uuid-integration into autopkgtest-cloud:master.

diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/run-autopkgtest b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/run-autopkgtest
index 2584323..48acc64 100755
--- a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/run-autopkgtest
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/run-autopkgtest
@@ -8,6 +8,7 @@ import json
 import os
 import sys
 import urllib.parse
+import uuid
 from datetime import datetime
 
 import amqplib.client_0_8 as amqp
@@ -176,6 +177,7 @@ if __name__ == "__main__":
     params["submit-time"] = datetime.strftime(
         datetime.utcnow(), "%Y-%m-%d %H:%M:%S%z"
     )
+    params["uuid"] = str(uuid.uuid1())
     params = "\n" + json.dumps(params)
 
     try:
diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker
index 3744f06..ef2ca83 100755
--- a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker
@@ -24,6 +24,7 @@ import sys
 import tempfile
 import time
 import urllib.request
+import uuid
 from urllib.error import HTTPError
 
 import amqplib.client_0_8 as amqp
@@ -322,7 +323,7 @@ def request_matches_per_package(package, arch, release, s):
 
 
 def process_output_dir(
-    dir, pkgname, code, triggers, additional_params: dict = None
+    dir, pkgname, code, triggers, test_uuid, additional_params: dict = None,
 ):
     """Post-process output directory"""
 
@@ -352,15 +353,17 @@ def process_output_dir(
                 d = {"custom_environment": ["ADT_TEST_TRIGGERS=%s" % triggers]}
                 json.dump(d, testinfo, indent=True)
             files.add("testinfo.json")
-
+    
+    # We have to add the uuid to testinfo.json
+    d = {}
+    with open(os.path.join(dir, "testinfo.json"), "r") as testinfo:
+        d = json.load(testinfo)
+    d["uuid"] = test_uuid
     if additional_params not in [None, {}]:
-        d = {}
-        with open(os.path.join(dir, "testinfo.json"), "r") as testinfo:
-            d = json.load(testinfo)
         for key, val in additional_params.items():
             d[key] = val
-        with open(os.path.join(dir, "testinfo.json"), "w") as testinfo:
-            json.dump(d, testinfo, indent=True)
+    with open(os.path.join(dir, "testinfo.json"), "w") as testinfo:
+        json.dump(d, testinfo, indent=True)
 
     with open(os.path.join(dir, "testpkg-version"), "r") as tpv:
         testpkg_version = tpv.read().split()[1]
@@ -438,7 +441,7 @@ def host_arch(release, architecture):
     return "amd64"
 
 
-def subst(s, big_package, release, architecture, hostarch, pkgname):
+def subst(s, big_package, release, architecture, hostarch, pkgname, test_uuid):
     subst = {
         "RELEASE": release,
         "ARCHITECTURE": architecture,
@@ -450,6 +453,7 @@ def subst(s, big_package, release, architecture, hostarch, pkgname):
         ),
         "TIMESTAMP": time.strftime("%Y%m%d-%H%M%S"),
         "HOSTNAME": hostname,
+        "UUID": test_uuid,
     }
     for i in args.variable:
         k, v = i.split("=", 1)
@@ -469,6 +473,7 @@ def send_status_info(
     out_dir,
     running,
     duration,
+    test_uuid,
     private=False,
 ):
     """Send status and logtail to status queue"""
@@ -505,6 +510,7 @@ def send_status_info(
             "params": params,
             "duration": duration,
             "logtail": logtail,
+            "uuid": test_uuid,
         }
     )
     queue.basic_publish(
@@ -520,6 +526,7 @@ def call_autopkgtest(
     params,
     out_dir,
     start_time,
+    test_uuid,
     private=False,
 ):
     """Call autopkgtest and regularly send status/logtail to status_exchange_name
@@ -554,6 +561,7 @@ def call_autopkgtest(
                 out_dir,
                 True,
                 int(time.time() - start_time),
+                test_uuid,
                 private,
             )
 
@@ -567,6 +575,7 @@ def call_autopkgtest(
         out_dir,
         False,
         int(time.time() - start_time),
+        test_uuid,
         private,
     )
 
@@ -641,7 +650,7 @@ def request(msg):
     except (ValueError, IndexError):
         logging.error('Received invalid request format "%s"', body)
         return
-
+    test_uuid = params.get("uuid", str(uuid.uuid1()))
     if not re.match("[a-zA-Z0-9.+-]+$", pkgname):
         logging.error(
             'Request contains invalid package name, dropping: "%s"', body
@@ -763,6 +772,7 @@ def request(msg):
                 architecture,
                 host_arch(release, architecture),
                 pkgname,
+                test_uuid,
             )
             argv += ["--setup-commands", c]
         c = cfg.get("autopkgtest", "setup_command2").strip()
@@ -774,6 +784,7 @@ def request(msg):
                 architecture,
                 host_arch(release, architecture),
                 pkgname,
+                test_uuid,
             )
             argv += ["--setup-commands", c]
 
@@ -1037,6 +1048,7 @@ def request(msg):
             argv.append("--testname=%s" % params["testname"])
 
         argv.append("--")
+        # Need to bake the UUID into these args! I think maybe the server-name
         argv += subst(
             cfg.get("virt", "args"),
             big_pkg,
@@ -1044,6 +1056,7 @@ def request(msg):
             architecture,
             host_arch(release, architecture),
             pkgname,
+            test_uuid,
         ).split()
 
         if "swiftuser" in params:
@@ -1078,9 +1091,9 @@ def request(msg):
                     params,
                     out_dir,
                     start_time,
+                    test_uuid,
                     private,
                 )
-
                 is_failure = code in FAIL_CODES
                 files = set(os.listdir(out_dir))
                 is_unknown_version = "testpkg-version" not in files
@@ -1263,6 +1276,7 @@ def request(msg):
             pkgname,
             code,
             params.get("triggers", []),
+            test_uuid,
             additional_parameters_for_testinfo,  # embed additional parameters in testinfo.json
         )
 
@@ -1387,6 +1401,7 @@ def request(msg):
             "swift_dir": swift_dir,
             "triggers": triggers,
             "env": ",".join(additional_parameters),
+            "uuid": test_uuid,
         }
     )
     complete_amqp.basic_publish(
@@ -1419,7 +1434,7 @@ def amqp_connect(cfg, callback):
     queue.basic_qos(0, 1, True)
 
     arch_str = cfg.get("autopkgtest", "architectures")
-    arch_str = subst(arch_str, "n/a", "n/a", "n/a", "n/a", "n/a")
+    arch_str = subst(arch_str, "n/a", "n/a", "n/a", "n/a", "n/a", "n/a")
     arches = arch_str.split()
     if not arches:
         my_arch = subprocess.check_output(
diff --git a/charms/focal/autopkgtest-web/webcontrol/amqp-status-collector b/charms/focal/autopkgtest-web/webcontrol/amqp-status-collector
index ed117fc..a61a949 100755
--- a/charms/focal/autopkgtest-web/webcontrol/amqp-status-collector
+++ b/charms/focal/autopkgtest-web/webcontrol/amqp-status-collector
@@ -67,6 +67,7 @@ def process_message(msg):
     params = info.get("params", {})
     for p in sorted(params):
         runhash += "%s_%s;" % (p, params[p])
+    params["uuid"] = info.get("uuid", "")
 
     if info["running"]:
         running_tests.setdefault(info["package"], {}).setdefault(
diff --git a/charms/focal/autopkgtest-web/webcontrol/browse.cgi b/charms/focal/autopkgtest-web/webcontrol/browse.cgi
index 4772be2..f39b30f 100755
--- a/charms/focal/autopkgtest-web/webcontrol/browse.cgi
+++ b/charms/focal/autopkgtest-web/webcontrol/browse.cgi
@@ -296,7 +296,7 @@ def package_release_arch(package, release, arch, _=None):
     seen = set()
     results = []
     for row in db_con.execute(
-        "SELECT run_id, version, triggers, duration, exitcode, requester, env FROM result "
+        "SELECT run_id, version, triggers, duration, exitcode, requester, env, uuid FROM result "
         "WHERE test_id=? "
         "ORDER BY run_id DESC",
         (test_id,),
@@ -339,6 +339,7 @@ def package_release_arch(package, release, arch, _=None):
                 url,
                 show_retry,
                 all_proposed,
+                row[7],
             )
         )
 
diff --git a/charms/focal/autopkgtest-web/webcontrol/download-all-results b/charms/focal/autopkgtest-web/webcontrol/download-all-results
index 92dfa0a..97f40a5 100755
--- a/charms/focal/autopkgtest-web/webcontrol/download-all-results
+++ b/charms/focal/autopkgtest-web/webcontrol/download-all-results
@@ -131,6 +131,7 @@ def fetch_one_result(url):
             testinfo = json.loads(
                 tar.extractfile("testinfo.json").read().decode()
             )
+            test_uuid = testinfo.get("uuid", "")
             duration = int(tar.extractfile("duration").read().strip())
             # KeyError means the file is not there, i.e. there isn't a human
             # requester
@@ -183,7 +184,7 @@ def fetch_one_result(url):
     try:
         c = db_con.cursor()
         c.execute(
-            "INSERT INTO result VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+            "INSERT INTO result VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
             (
                 test_id,
                 run_id,
@@ -193,6 +194,7 @@ def fetch_one_result(url):
                 exitcode,
                 requester,
                 ",".join(env_vars),
+                test_uuid,
             ),
         )
         db_con.commit()
diff --git a/charms/focal/autopkgtest-web/webcontrol/download-results b/charms/focal/autopkgtest-web/webcontrol/download-results
index c36b5ab..1b975bf 100755
--- a/charms/focal/autopkgtest-web/webcontrol/download-results
+++ b/charms/focal/autopkgtest-web/webcontrol/download-results
@@ -56,6 +56,7 @@ def process_message(msg, db_con):
     release = info["release"]
     requester = (info["requester"] or "").strip()
     version = info["testpkg_version"]
+    test_uuid = info["uuid"]
     (_, _, _, _, run_id) = info["swift_dir"].split("/")
 
     # we don't handle PPA requests and their container names are
@@ -84,7 +85,7 @@ def process_message(msg, db_con):
     try:
         c = db_con.cursor()
         c.execute(
-            "INSERT INTO result VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+            "INSERT INTO result VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
             (
                 test_id,
                 run_id,
@@ -94,6 +95,7 @@ def process_message(msg, db_con):
                 exitcode,
                 requester,
                 info.get("env", ""),
+                test_uuid,
             ),
         )
         db_con.commit()
diff --git a/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py b/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
index 8ad9eeb..11357fc 100644
--- a/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
+++ b/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
@@ -44,6 +44,7 @@ def init_db(path):
             "  exitcode INTEGER, "
             "  requester TEXT, "
             "  env TEXT, "
+            "  uuid TEXT,  "
             "  PRIMARY KEY(test_id, run_id), "
             "  FOREIGN KEY(test_id) REFERENCES test(id))"
         )
diff --git a/charms/focal/autopkgtest-web/webcontrol/request/submit.py b/charms/focal/autopkgtest-web/webcontrol/request/submit.py
index 3466254..4f7bd77 100644
--- a/charms/focal/autopkgtest-web/webcontrol/request/submit.py
+++ b/charms/focal/autopkgtest-web/webcontrol/request/submit.py
@@ -12,6 +12,7 @@ import re
 import sqlite3
 import urllib.parse
 import urllib.request
+import uuid
 from datetime import datetime
 from urllib.error import HTTPError
 
@@ -336,6 +337,7 @@ class Submit:
         params["submit-time"] = datetime.strftime(
             datetime.utcnow(), "%Y-%m-%d %H:%M:%S%z"
         )
+        params["uuid"] = str(uuid.uuid1())
         body = "%s %s" % (package, json.dumps(params, sort_keys=True))
         with amqp.Connection(
             self.amqp_creds.hostname,
diff --git a/charms/focal/autopkgtest-web/webcontrol/templates/browse-results.html b/charms/focal/autopkgtest-web/webcontrol/templates/browse-results.html
index 3da1e24..e5cbc2a 100644
--- a/charms/focal/autopkgtest-web/webcontrol/templates/browse-results.html
+++ b/charms/focal/autopkgtest-web/webcontrol/templates/browse-results.html
@@ -11,6 +11,7 @@
       <td><b>Duration</b></td>
       <td><b>Requester</b></td>
       <td><b>Result</b></td>
+      <td><b>uuid</b></td>
       <td></td>
     </tr>
 
@@ -23,7 +24,7 @@
       <td class="{{row[4]}}">{{row[4]}}</td>
       <td>
         {% if row[5] != "-" %}
-          <a href="https://launchpad.net/~{{row[4]}}";>{{row[4]}}</a>
+          <a href="https://launchpad.net/~{{row[5]}}";>{{row[5]}}</a>
         {% else %}
           {{row[5]}}
         {% endif %}
@@ -31,6 +32,9 @@
       <td class="nowrap">
         <a href="{{row[7]}}/log.gz">log</a> &emsp;
         <a href="{{row[7]}}/artifacts.tar.gz">artifacts</a> &emsp;
+      </td>
+      <td>{{row[10]}}</td>
+      <td class="nowrap">
         {% if row[8] %}
           {% set ts = row[1].split()|map('urlencode')|join("&trigger=")|safe %}
           {% set all_proposed = row[9] %}
diff --git a/charms/focal/autopkgtest-web/webcontrol/templates/browse-running.html b/charms/focal/autopkgtest-web/webcontrol/templates/browse-running.html
index 2d0966b..3711b97 100644
--- a/charms/focal/autopkgtest-web/webcontrol/templates/browse-running.html
+++ b/charms/focal/autopkgtest-web/webcontrol/templates/browse-running.html
@@ -47,6 +47,8 @@
             {% for param, v in params.items() %}
               {% if param == "requester" %}
             <tr><th>{{param|capitalize}}:</th><td><a href="https://launchpad.net/~{{v}}";>{{v}}</a></td></tr>
+              {% elif param == "uuid" %}
+            <tr><th>{{param|capitalize}}:</th><td>{{v}}</td></tr>
               {% else %}
             <tr><th>{{param|capitalize}}:</th><td>{{v}}</td></tr>
               {% endif %}
diff --git a/mojo/service-bundle b/mojo/service-bundle
index ad86421..ce19fa9 100644
--- a/mojo/service-bundle
+++ b/mojo/service-bundle
@@ -90,11 +90,11 @@ applications:
 {%- endif %}
 {%- if stage_name == "production" or stage_name == "staging" %}
             mirror: http://ftpmaster.internal/ubuntu/
-            worker-args: ssh -s /CHECKOUTDIR//ssh-setup/nova -- --flavor $PACKAGESIZE --security-groups $SECGROUP --name adt-$RELEASE-$ARCHITECTURE-$PACKAGENAME-$TIMESTAMP-$HOSTNAME --image adt/ubuntu-$RELEASE-$HOSTARCH-server --keyname testbed-/HOSTNAME/ --net-id=/NET_NAME/ -e TERM=linux -e 'http_proxy={{ http_proxy }}' -e 'https_proxy={{ https_proxy }}' -e 'no_proxy={{ no_proxy }}' --mirror=/MIRROR/
+            worker-args: ssh -s /CHECKOUTDIR//ssh-setup/nova -- --flavor $PACKAGESIZE --security-groups $SECGROUP --name adt-$RELEASE-$ARCHITECTURE-$PACKAGENAME-$TIMESTAMP-$HOSTNAME-$UUID --image adt/ubuntu-$RELEASE-$HOSTARCH-server --keyname testbed-/HOSTNAME/ --net-id=/NET_NAME/ -e TERM=linux -e 'http_proxy={{ http_proxy }}' -e 'https_proxy={{ https_proxy }}' -e 'no_proxy={{ no_proxy }}' --mirror=/MIRROR/
             worker-setup-command: /AUTOPKGTEST_CLOUD_DIR//worker-config-production/setup-canonical.sh
 {% else %}
             mirror: http://ports.ubuntu.com/ubuntu-ports/
-            worker-args: ssh -s /CHECKOUTDIR//ssh-setup/nova -- --flavor $PACKAGESIZE --security-groups $SECGROUP --name adt-$RELEASE-$ARCHITECTURE-$PACKAGENAME-$TIMESTAMP-$HOSTNAME --image adt/ubuntu-$RELEASE-$ARCHITECTURE-server-.* --keyname testbed-/HOSTNAME/ --mirror=/MIRROR/
+            worker-args: ssh -s /CHECKOUTDIR//ssh-setup/nova -- --flavor $PACKAGESIZE --security-groups $SECGROUP --name adt-$RELEASE-$ARCHITECTURE-$PACKAGENAME-$TIMESTAMP-$HOSTNAME-$UUID --image adt/ubuntu-$RELEASE-$ARCHITECTURE-server-.* --keyname testbed-/HOSTNAME/ --mirror=/MIRROR/
 {% endif %}
 {%- if stage_name == "production" %}
             n-workers: |-