canonical-ubuntu-qa team mailing list archive

[Tim Andersson <mp+480247@xxxxxxxxxxxxxxxxxx>]

Tim Andersson has proposed merging qa-jenkins-jobs:hardware-installer-testing-refactor into qa-jenkins-jobs:master.

Requested reviews:
  Canonical Platform QA Team (canonical-platform-qa)

For more details, see:
https://code.launchpad.net/~canonical-platform-qa/qa-jenkins-jobs/+git/qa-jenkins-jobs/+merge/480247

Refactors the hardware-installer-testing jobs
-- 
Your team Canonical Platform QA Team is requested to review the proposed merge of qa-jenkins-jobs:hardware-installer-testing-refactor into qa-jenkins-jobs:master.

diff --git a/jobs/hardware-iso-testing/jobs.yaml b/jobs/hardware-iso-testing/jobs.yaml
index 91d2d2a..54365e9 100644
--- a/jobs/hardware-iso-testing/jobs.yaml
+++ b/jobs/hardware-iso-testing/jobs.yaml
@@ -18,78 +18,127 @@
 
 # The project stanza describes all jobs and parameters
 
+
+# Future improvements:
+# - throttle concurrent builds
+# - lockable resources
+# - exclusion
+
 - project:
     name: 'hardware-iso-testing'
     release:
-      - 'noble'
-    dut-id:
-      - '202101-28611'
-      - '202207-30464'
-      - '202203-30106'
+        - 'noble'
+    dut_id:
+        # - '202101-28611'  # dell optiplex 5090 - inconsistent machine, crashes sometimes
+        - '202207-30464'  # hp pro sff 400 g9 - also gone AWOL this morn 28/01/2025 :(
+        # - '202203-30106'  # dell precision 7865 - connection refused
+        # - '202102-28694'  # lenovo thinkstation p350 - No signal after booting from usb - could be nvidia driver issue
+    test_case:
+        - 'erase-and-reinstall'
+        - 'entire-disk-with-zfs'
+        # - 'entire-disk-with-zfs-plus-encryption'  # needs wait_for_text
+        - 'entire-disk-with-installer-update'
+        # - 'entire-disk-with-lvm-and-encryption'  # needs wait_for_text
+        - 'entire-disk-no-internet'
+        - 'oem-install'
+        - 'entire-disk-with-nvidia-drivers'
     jobs:
-      - 'desktop-hardware-installer-tests-{release}-{dut-id}'
+        - 'hit-{release}-dut-test-case-mapping'
+        - 'hit-{release}-desktop-{test_case}'
 
-# builder stanzas describe reusable build steps
-- builder:
-    name: run-hardware-installer-test
+- job-template:
+    name: 'hit-{release}-dut-test-case-mapping'
+    description: |
+        Maps a test case to a DUT id - preferring to do so in a manner to parallelise as much as poss
+    parameters:
+        - string:
+            name: 'DUT_IDS'
+            description: 'Space separated list of c3 DUT ids'
+            default:
+                !j2: '{{ " ".join(dut_id) }}'
+        - string:
+            name: 'TEST_CASES'
+            description: 'Space separated list of test case names'
+            default: 
+                !j2: '{{ " ".join(test_case) }}'
+    wrappers:
+      - timestamps
+      - timeout:
+          timeout: 15
+          fail: true
     builders:
-      - shell: |
-            set -e
-            retry -t 3 -d 180 -- git clone -b "${{HW_INSTALLER_TESTING_BRANCH}}" --depth 1 https://github.com/canonical/hardware-installer-testing.git hardware-installer-testing
-            retry -t 3 -d 180 -- git clone --depth 1 https://github.com/canonical/testflinger
-            python3 -m venv env
-            . env/bin/activate
-            pip install testflinger/cli
-            pip install -r hardware-installer-testing/requirements.txt
-            cd hardware-installer-testing/
-            development_release=$(distro-info --devel || echo UNKNOWN)
-            if [ {release} = $development_release ]; then
-                # iso_url="https://tel-image-cache.canonical.com/cdimage/daily-live/current/{release}-desktop-amd64.iso";
-                iso_url="https://cdimage.ubuntu.com/daily-live/current/{release}-desktop-amd64.iso";
-            else
-                # iso_url="https://tel-image-cache.canonical.com/cdimage/{release}/daily-live/current/{release}-desktop-amd64.iso";
-                iso_url="https://cdimage.ubuntu.com/{release}/daily-live/current/{release}-desktop-amd64.iso";
-            fi
-            python3 ./scripts/call_testflinger_job.py --job-config test-definitions/{release}/$TEST_CASE.json --c3-machine-id {dut-id} --iso-url $iso_url
+        - python: |
+            import os
+            import json
+            import random
+            import sys
+            duts = os.environ.get("DUT_IDS").split(" ")
+            test_cases = os.environ.get("TEST_CASES").split(" ")
+            if not duts or not test_cases:
+                sys.exit(1)
+            used_duts = []
+            op_json = {{}}
+            while len(test_cases) > 0:
+                if len(duts) == 0:
+                    duts = used_duts
+                    used_duts = []
+                duts = random.shuffle(duts)
+                idx = random.randrange(len(duts))
+                op_json[test_cases[0]] = duts[idx]
+                used_duts.append(duts[idx])
+                del duts[idx]
+                del test_cases[0]
+            with open("{release}-dut-test-case-mapping.json", "w") as f:
+                json.dump(op_json, f)
+    publishers:
+        - archive:
+            artifacts: '{release}-dut-test-case-mapping.json'
 
-# job templates
 - job-template:
-    name: 'desktop-hardware-installer-tests-{release}-{dut-id}'
-    project-type: matrix
-    axes:
-      - axis:
-          type: user-defined
-          name: TEST_CASE
-          values:
-            - erase-and-reinstall
-            - entire-disk-with-zfs
-            # - entire-disk-with-zfs-plus-encryption
-            - entire-disk-with-installer-update
-            # - entire-disk-with-lvm-and-encryption
-            - entire-disk-no-internet
-            - oem-install
-            - entire-disk-with-nvidia-drivers
-    execution-strategy:
-      sequential: true
-      touchstone:
-        expr: 'TEST_CASE == "erase-and-reinstall"'
+    name: 'hit-{release}-desktop-{test_case}'
     description: |
-        Runs a test defined in gh:/canonical/hardware-installer-testing
+        Runs a test case on a DUT as specified by dut-test-case-mapping
+    triggers:
+        - reverse:
+            jobs: '{release}-dut-test-case-mapping'
+            result: success
     parameters:
       - string:
-          name: HW_INSTALLER_TESTING_BRANCH
-          default: main
-          description: "github branch of hardware-installer-testing from which to run the tests from."
+          name: HIT_BRANCH
+          default: 'installer-test-suite'
+          description: branch of HIT to run tests from
     wrappers:
-      - timeout:
-          timeout: 10800
-          fail: true
       - timestamps
+      - credentials-binding:
+          - text:
+              credential-id: GH_PAT
+              variable: GH_PAT
     builders:
-      - clear-artifacts:
-      - run-hardware-installer-test:
-          release: '{release}'
-          dut-id: '{dut-id}'
-    publishers:
-      - archive:
-          artifacts: 'hardware-installer-testing/artifacts/*'
+        - copyartifact:
+            project: '{release}-dut-test-case-mapping'
+            filter: '{release}-dut-test-case-mapping.json'
+            which-build: last-successful
+        - shell: |
+            #!/bin/bash
+            # get DUT_ID
+            DUT_ID=$(jq -r '."{test_case}"' dut-test-case-mapping)
+            # clone HIT and testflinger
+            retry -t 3 -d 180 -- git clone -b $HIT_BRANCH https://oauth2:$GH_PAT@xxxxxxxxxx/canonical/hardware-installer-testing.git
+            retry -t 3 -d 180 -- git clone https://github.com/canonical/testflinger
+            # set up venv
+            python3 -m venv env
+            . env/bin/activate
+            pip install testflinger/cli
+            pip install -r hardware-installer-testing/requirements.txt
+            # determine ISO url
+            development_release=$(distro-info --devel || echo UNKNOWN)
+            if [ {release} = $development_release ]; then
+                ISO_URL="https://tel-image-cache.canonical.com/cdimage/daily-live/current/{release}-desktop-amd64.iso";
+            else
+                ISO_URL="https://tel-image-cache.canonical.com/cdimage/{release}/daily-live/current/{release}-desktop-amd64.iso";
+            fi
+            # call job!
+            # Like this for now, just to test!
+            cd hardware-installer-testing/
+            echo "Would call:"
+            echo "./scripts/call_testflinger_job_w_yarf.py --test-case {test_case} --c3-machine-id $DUT_ID --iso-url $ISO_URL --github-pat-token $GH_PAT --hit-branch $HIT_BRANCH --yarf-branch main"