canonical-ubuntu-qa team mailing list archive

[Tim Andersson <mp+465334@xxxxxxxxxxxxxxxxxx>]

Tim Andersson has proposed merging ~andersson123/autopkgtest-cloud:proposed-package-images into autopkgtest-cloud:master.

Commit message:
fix: cloud: make create-nova-image-with-proposed-package up to date

This commit introduces a new mechanism of loading creds for the
aforementioned script, given that we now use a wider variety of
datacentres, and this script was last updated nearly 5 years ago.

This script also adds two new dependencies to the cloud-worker charm:
- qemu-user-static
- binfmt-support
These are required for the script to execute bash commands on vm's on a
different arch to the host VM - i.e. on arm64 from an amd64 host.

It also modifies the mechanism in which the desired package is installed
from proposed, and does away with the sed line that existed before.

create-nova-image-with-proposed is a script, which hasn't been used in
*quite* a while, which rebuilds one of our adt images, with a specified
package from proposed.

We had to use this recently when the version of base-files in the
release pocket was breaking our tests, but the version of base-files
in the proposed pocket would fix said issue.

Requested reviews:
  Paride Legovini (paride)
  Canonical's Ubuntu QA (canonical-ubuntu-qa)

For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/465334

This was used recently because we needed to install base-files from -proposed when building the oracular images.

Tim found the script was incompatible with our current versions of glance that we're using, and he had to update the script in order to rebuild adt images with the version of base-files that was in proposed.
-- 
Your team Canonical's Ubuntu QA is requested to review the proposed merge of ~andersson123/autopkgtest-cloud:proposed-package-images into autopkgtest-cloud:master.

diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/create-nova-image-with-proposed-package b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/create-nova-image-with-proposed-package
index e48f3f4..2dc585f 100755
--- a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/create-nova-image-with-proposed-package
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/create-nova-image-with-proposed-package
@@ -22,23 +22,35 @@ import tempfile
 
 import glanceclient
 import keystoneauth1.loading
+from distro_info import UbuntuDistroInfo
 from glanceclient.common import utils
 
 
 def get_glance():
     """Return glance client object"""
-
     loader = keystoneauth1.loading.get_plugin_loader("password")
-    auth = loader.load_from_options(
-        auth_url=os.environ["OS_AUTH_URL"],
-        username=os.environ["OS_USERNAME"],
-        password=os.environ["OS_PASSWORD"],
-        tenant_name=os.environ["OS_TENANT_NAME"],
-    )
+    if os.environ.get("OS_IDENTITY_API_VERSION") == "3":
+        auth = loader.load_from_options(
+            auth_url=os.environ["OS_AUTH_URL"],
+            username=os.environ["OS_USERNAME"],
+            password=os.environ["OS_PASSWORD"],
+            tenant_name=os.environ["OS_TENANT_NAME"],
+            project_name=os.environ["OS_PROJECT_NAME"],
+            user_domain_name=os.environ["OS_USER_DOMAIN_NAME"],
+            project_domain_name=os.environ["OS_PROJECT_DOMAIN_NAME"],
+        )
+    else:
+        auth = loader.load_from_options(
+            auth_url=os.environ["OS_AUTH_URL"],
+            username=os.environ["OS_USERNAME"],
+            password=os.environ["OS_PASSWORD"],
+            tenant_name=os.environ["OS_TENANT_NAME"],
+        )
     session = keystoneauth1.session.Session(auth=auth)
-    return glanceclient.Client(
-        "2", session=session, region_name=os.environ["NOVA_REGION"]
+    os_region = os.environ.get(
+        "NOVA_REGION", os.environ.get("OS_REGION_NAME", "")
     )
+    return glanceclient.Client("2", session=session, region_name=os_region)
 
 
 def find_latest_image(img_re):
@@ -55,7 +67,7 @@ def find_latest_image(img_re):
     return latest
 
 
-def setup_image(image_path, source):
+def setup_image(image_path, source, release):
     # get a chroot shell into the image
     img_shell = subprocess.Popen(
         [
@@ -76,30 +88,36 @@ def setup_image(image_path, source):
     # pylint: disable=line-too-long
     img_shell.stdin.write(
         (
-            """
+            f"""
 set -e
 echo '* Creating policy-rc.d'
 printf '#!/bin/sh\\nexit 101\\n' > /usr/sbin/policy-rc.d
 chmod 755 /usr/sbin/policy-rc.d
 echo '* Generating apt sources for -proposed:'
-sed -rn 's/^(deb|deb-src) +(\[.*\] *)?([^ ]*(ubuntu.com|debian.org|ftpmaster)[^ ]*) +([^ -]+) +(.*)$/\\1 \\2\\3 \\5-proposed \\6/p' /etc/apt/sources.list `ls /etc/apt/sources.list.d/*.list 2>/dev/null|| true` > /etc/apt/sources.list.d/proposed.list
+
+echo "deb     http://ftpmaster.internal/ubuntu/ {release}-proposed main restricted universe multiverse" > /etc/apt/sources.list.d/proposed.list
+echo "deb-src http://ftpmaster.internal/ubuntu/ {release}-proposed main restricted universe multiverse" >> /etc/apt/sources.list.d/proposed.list
+
 cat /etc/apt/sources.list.d/proposed.list
 
 echo '* apt-get update'
-apt-get update
+apt-get -o "Dir::Etc::sourcelist=/etc/apt/sources.list.d/proposed.list" update
 
-echo '* Determining which binaries of source %(src)s are installed and have a new version...'
-SRCS=$(python3 -c 'import sys, apt; src=sys.argv[1]; [print(pkg.name) for pkg in apt.Cache() if pkg.installed and pkg.candidate.source_name == src and pkg.candidate.version > pkg.installed.version]' %(src)s)
+echo '* Determining which binaries of source {source} are installed and have a new version...'
+SRCS=$(python3 -c 'import sys, apt; src=sys.argv[1]; [print(pkg.name) for pkg in apt.Cache() if pkg.installed and pkg.candidate.source_name == src and pkg.candidate.version > pkg.installed.version]' {source})
 echo "$SRCS"
 
-echo '* Install the above packages'
+-echo '* Install the above packages'
 DEBIAN_FRONTEND=noninteractive apt-get install -y $SRCS
 
-echo '* Cleaning up'
+echo '* Cleaning up apt'
+rm /etc/apt/sources.list.d/proposed.list
+apt-get update
 apt-get clean
+
+echo '* Cleaning up'
 rm -f /etc/machine-id /usr/sbin/policy-rc.d
    """
-            % {"src": source}
         ).encode()
     )
 
@@ -117,10 +135,22 @@ if len(sys.argv) != 3:
     )
     sys.exit(1)
 
+udi = UbuntuDistroInfo()
+releases = set(udi.supported() + udi.supported_esm())
 image_re = re.compile(sys.argv[1])
-source = sys.argv[2]
 glance = get_glance()
 latest = find_latest_image(image_re)
+source = sys.argv[2]
+release = None
+for r in releases:
+    if r in latest["name"]:
+        print(f"* Parsed release {r} from image {latest['name']}")
+        release = r
+if release is None:
+    sys.stderr.write(
+        f"* Couldn't parse any release from image {latest['name']}, exiting..."
+    )
+    sys.exit(1)
 
 print("* Downloading image %s (UUID: %s)..." % (latest.name, latest.id))
 workdir = tempfile.TemporaryDirectory(
@@ -129,8 +159,9 @@ workdir = tempfile.TemporaryDirectory(
 img = os.path.join(workdir.name, "image")
 utils.save_image(glance.images.data(latest.id), img)
 
-setup_image(img, source)
-
+print("* Image %s has been downloaded - now setting up")
+setup_image(img, source, release)
+print("* Image has been set up! Creating image via glance API")
 newimg_name = "proposed-%s/%s" % (source, os.path.basename(latest.name))
 newimg = glance.images.create(
     name=newimg_name,
@@ -140,3 +171,4 @@ newimg = glance.images.create(
 print("* Uploading new image %s (UUID: %s)..." % (newimg.name, newimg.id))
 with open(img, "rb") as f:
     glance.images.upload(newimg.id, f)
+print("* Image uploaded! All done.")
diff --git a/charms/focal/autopkgtest-cloud-worker/layer.yaml b/charms/focal/autopkgtest-cloud-worker/layer.yaml
index 70be935..a150607 100644
--- a/charms/focal/autopkgtest-cloud-worker/layer.yaml
+++ b/charms/focal/autopkgtest-cloud-worker/layer.yaml
@@ -30,4 +30,6 @@ options:
       - python3-swiftclient
       - ssmtp
       - vim
+      - qemu-user-static
+      - binfmt-support
     include_system_packages: true