launchpad-reviewers team mailing list archive

[Colin Watson <mp+452644@xxxxxxxxxxxxxxxxxx>]

Colin Watson has proposed merging ~cjwatson/launchpad-mojo-specs/+git/private:vbuilder-qastaging into ~launchpad/launchpad-mojo-specs/+git/private:vbuilder with ~cjwatson/launchpad-mojo-specs/+git/private:vbuilder-remove-lgw01 as a prerequisite.

Commit message:
vbuilder: Add qastaging

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~cjwatson/launchpad-mojo-specs/+git/private/+merge/452644

This shares OpenStack credentials with staging, but uses different instance networks, domains, image names, and builder names so that things don't overlap where that would cause problems.  This is working well on qastaging already so I'd like to merge it back.

I took the opportunity to rewrite `vbuilder/upgrade-charms` to look a bit more like `vbuilder/rebuild-images`, as it was getting hard to understand what to change here and it was easy to miss things.
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad-mojo-specs/+git/private:vbuilder-qastaging into ~launchpad/launchpad-mojo-specs/+git/private:vbuilder.

diff --git a/vbuilder/bundle.yaml b/vbuilder/bundle.yaml
index 6a6c5e3..d588d9d 100644
--- a/vbuilder/bundle.yaml
+++ b/vbuilder/bundle.yaml
@@ -41,7 +41,8 @@
 {%-   set vbuilders_bos03 = {"amd64": {"series": "focal", "count": 60}} %}
 {%-   set vbuilders_lcy02 = {"amd64": {"series": "focal", "count": 120}} %}
 {%-   set vbuilder_prefix = "" %}
-{%- else %}
+{%- elif stage_name == "staging" %}
+{#-   This environment is confusingly named, and is actually connected to Launchpad dogfood. #}
 {%-   set clamav_database_url = "http://clamav-database-mirror.staging.lp.internal/"; %}
 {%-   set content_id_template = "launchpad-buildd:staging" %}
 {%-   set dns_update_host_bos01 = "10.189.0.2" %}
@@ -84,6 +85,49 @@
 {%-   set vbuilders_bos03 = {"amd64": {"series": "jammy", "count": 4}} %}
 {%-   set vbuilders_lcy02 = {"amd64": {"series": "jammy", "count": 4}} %}
 {%-   set vbuilder_prefix = "dogfood-" %}
+{%- elif stage_name == "qastaging" %}
+{%-   set clamav_database_url = "http://clamav-database-mirror.staging.lp.internal/"; %}
+{%-   set content_id_template = "launchpad-buildd:qastaging" %}
+{%-   set dns_update_host_bos01 = "10.189.0.2" %}
+{%-   set dns_update_host_bos02 = "10.189.128.2" %}
+{%-   set dns_update_host_bos03 = "10.189.128.2" %}
+{%-   set dns_update_host_lcy02 = "10.132.31.11 10.132.31.12 10.132.31.13" %}
+{%-   set dns_update_key_name = "vbuilder-staging-manage" %}
+{%-   set domain_bos01 = "vbuilder.qastaging.bos01.scalingstack" %}
+{%-   set domain_bos02 = "vbuilder.qastaging.bos02.scalingstack" %}
+{%-   set domain_bos03 = "vbuilder.qastaging.bos03.scalingstack" %}
+{%-   set domain_lcy02 = "vbuilder.qastaging.lcy02.scalingstack" %}
+{%-   set extra_constraints = "" %}
+{%-   set gss_series = "focal|jammy" %}
+{%-   set instance_key_name_bos01 = "ppa-manage-test-qastaging" %}
+{%-   set instance_key_name_bos02 = "ppa-manage-test-qastaging" %}
+{%-   set instance_key_name_bos03 = "ppa-manage-test-qastaging" %}
+{%-   set instance_key_name_lcy02 = "ppa-manage-test-qastaging" %}
+{%-   set instance_network_bos01 = "10.189.36.0/24" %}
+{%-   set instance_network_bos02 = "10.189.164.0/23" %}
+{%-   set instance_network_bos03 = "10.144.4.0/23" %}
+{%-   set instance_network_lcy02 = "10.134.4.0/23" %}
+{%-   set instance_router = "vbuilder_staging_router" %}
+{%-   set instance_router_bos03 = "router_launchpad-vbuilder-staging" %}
+{%-   set instance_router_lcy02 = "router_launchpad-vbuilder-staging" %}
+{%-   set launchpad_buildd_repository = "ppa:launchpad/buildd-staging" %}
+{%-   set lp_buildd_managers = "10.132.54.242" %}
+{%-   set lp_environment = "qastaging" %}
+{%-   set lp_sshkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFrjt0yytzrK9fQuG+6VgE6QStUbDmunlN7+Lv5XhmoL stg-launchpad@launchpad-bastion-ps5" %}
+{%-   set modifiers_bos01 = '{"arm64": "10.43.0.10", "ppc64el": "10.43.0.23", "s390x": "10.43.0.15"}' %}
+{%-   set modifiers_bos02 = '{"arm64": "10.44.0.13", "ppc64el": "10.44.0.19", "s390x": "10.44.0.14"}' %}
+{%-   set name_prefix = "launchpad-buildd-qastaging" %}
+{%-   set openstack_tenant_name = "vbuilder_staging_project" %}
+{%-   set openstack_tenant_name_bos03 = "launchpad-vbuilder-staging_project" %}
+{%-   set openstack_tenant_name_lcy02 = "launchpad-vbuilder-staging_project" %}
+{%-   set openstack_username = "vbuilder_staging" %}
+{%-   set openstack_username_bos03 = "launchpad-vbuilder-staging" %}
+{%-   set openstack_username_lcy02 = "launchpad-vbuilder-staging" %}
+{%-   set vbuilders_bos01 = {"amd64": {"series": "focal", "flavor": "vbuilder-gpu", "count": 1}, "arm64": {"series": "focal", "count": 1, "config_drive": false}, "arm64-gpu": {"arch_base": "arm64", "arch_suffix": "-gpu", "series": "focal", "flavor": "vbuilder-nvidia-l4", "count": 1, "config_drive": false}, "ppc64el": {"series": "focal", "count": 1}, "s390x": {"series": "focal", "count": 1}} %}
+{%-   set vbuilders_bos02 = {"arm64": {"series": "focal", "count": 1, "config_drive": false}, "ppc64el": {"series": "focal", "count": 1}, "s390x": {"series": "focal", "count": 1}} %}
+{%-   set vbuilders_bos03 = {"amd64": {"series": "focal", "count": 4}} %}
+{%-   set vbuilders_lcy02 = {"amd64": {"series": "focal", "count": 4}} %}
+{%-   set vbuilder_prefix = "qastaging-" %}
 {%- endif %}
 
 {%- macro vbuilder_hostnames(prefix, count) %}
diff --git a/vbuilder/rebuild-images b/vbuilder/rebuild-images
index 5246cd1..0f66c7f 100755
--- a/vbuilder/rebuild-images
+++ b/vbuilder/rebuild-images
@@ -6,11 +6,24 @@ from utils import utils
 
 
 name_prefix_by_stage = {
+    "qastaging": "launchpad-buildd-qastaging",
     "staging": "launchpad-buildd-staging",
     "production": "launchpad-buildd",
     }
 
 targets_by_stage = {
+    "qastaging": [
+        ("bos01", "amd64", "focal"),
+        ("bos01", "arm64", "focal"),
+        ("bos01", "arm64-gpu", "focal"),
+        ("bos01", "ppc64el", "focal"),
+        ("bos01", "s390x", "focal"),
+        ("bos02", "arm64", "focal"),
+        ("bos02", "ppc64el", "focal"),
+        ("bos02", "s390x", "focal"),
+        ("bos03", "amd64", "focal"),
+        ("lcy02", "amd64", "focal"),
+        ],
     "staging": [
         ("bos01", "amd64", "jammy"),
         ("bos01", "arm64", "jammy"),
diff --git a/vbuilder/upgrade-charms b/vbuilder/upgrade-charms
index 471b7c4..6f7f70d 100755
--- a/vbuilder/upgrade-charms
+++ b/vbuilder/upgrade-charms
@@ -5,58 +5,85 @@ import utils.check_version  # noqa: F401
 import os.path
 import subprocess
 
-from utils import utils
-
-
-juju_services = utils.juju_services()
-apps = [
-    ('glance-simplestreams-sync-bos01-amd64',
-     'glance-simplestreams-sync'),
-    ('glance-simplestreams-sync-bos01-arm64',
-     'glance-simplestreams-sync'),
-    ('glance-simplestreams-sync-bos01-ppc64el',
-     'glance-simplestreams-sync'),
-    ('glance-simplestreams-sync-bos01-s390x',
-     'glance-simplestreams-sync'),
-    ('glance-simplestreams-sync-bos02-arm64',
-     'glance-simplestreams-sync'),
-    ('glance-simplestreams-sync-bos02-ppc64el',
-     'glance-simplestreams-sync'),
-    ('glance-simplestreams-sync-bos02-s390x',
-     'glance-simplestreams-sync'),
-    ('glance-simplestreams-sync-bos03-amd64',
-     'glance-simplestreams-sync'),
-    ('glance-simplestreams-sync-lcy02-amd64',
-     'glance-simplestreams-sync'),
-    ('launchpad-buildd-image-modifier-amd64',
-     'launchpad-buildd-image-modifier'),
-    ('launchpad-buildd-image-modifier-bos01-amd64',
-     'launchpad-buildd-image-modifier'),
-    ('launchpad-buildd-image-modifier-bos01-arm64',
-     'launchpad-buildd-image-modifier'),
-    ('launchpad-buildd-image-modifier-bos01-ppc64el',
-     'launchpad-buildd-image-modifier'),
-    ('launchpad-buildd-image-modifier-bos01-s390x',
-     'launchpad-buildd-image-modifier'),
-    ('launchpad-buildd-image-modifier-bos02-arm64',
-     'launchpad-buildd-image-modifier'),
-    ('launchpad-buildd-image-modifier-bos02-ppc64el',
-     'launchpad-buildd-image-modifier'),
-    ('launchpad-buildd-image-modifier-bos02-s390x',
-     'launchpad-buildd-image-modifier'),
-    ('vbuilder-manage-bos02',
-     'vbuilder-manage'),
-    ('vbuilder-manage-bos03',
-     'vbuilder-manage'),
-    ('vbuilder-manage-lcy02',
-     'vbuilder-manage'),
-    ]
-
-for app_name, charm_name in apps:
-    if app_name not in juju_services:
-        continue
-    subprocess.check_call([
-        'juju', 'upgrade-charm', app_name,
-        '--path=%s' % os.path.join(
-            os.environ['MOJO_REPO_DIR'], os.environ['MOJO_SERIES'],
-            charm_name)])
+
+targets_by_stage = {
+    "qastaging": [
+        ("bos01", "amd64"),
+        ("bos01", "arm64"),
+        ("bos01", "arm64-gpu"),
+        ("bos01", "ppc64el"),
+        ("bos01", "s390x"),
+        ("bos02", "arm64"),
+        ("bos02", "ppc64el"),
+        ("bos02", "s390x"),
+        ("bos03", "amd64"),
+        ("lcy02", "amd64"),
+    ],
+    "staging": [
+        ("bos01", "amd64"),
+        ("bos01", "arm64"),
+        ("bos01", "arm64-gpu"),
+        ("bos01", "ppc64el"),
+        ("bos01", "s390x"),
+        ("bos02", "arm64"),
+        ("bos02", "ppc64el"),
+        ("bos02", "s390x"),
+        ("bos03", "amd64"),
+        ("lcy02", "amd64"),
+    ],
+    "production": [
+        ("bos01", "amd64"),
+        ("bos01", "arm64"),
+        ("bos01", "ppc64el"),
+        ("bos01", "s390x"),
+        ("bos02", "arm64"),
+        ("bos02", "ppc64el"),
+        ("bos02", "s390x"),
+        ("bos03", "amd64"),
+        ("lcy02", "amd64"),
+    ],
+}
+
+
+upgraded = set()
+
+
+def upgrade_charm(app_name, charm_name):
+    if app_name in upgraded:
+        return
+    subprocess.check_call(
+        [
+            "juju",
+            "upgrade-charm",
+            app_name,
+            "--path=%s"
+            % os.path.join(
+                os.environ["MOJO_REPO_DIR"],
+                os.environ["MOJO_SERIES"],
+                charm_name,
+            ),
+        ]
+    )
+    upgraded.add(app_name)
+
+
+def main():
+    stage = os.path.basename(os.environ["MOJO_STAGE"])
+    targets = targets_by_stage[stage]
+    for region, arch in targets:
+        upgrade_charm(
+            f"glance-simplestreams-sync-{region}-{arch}",
+            "glance-simplestreams-sync",
+        )
+        # lcy02-amd64 uses an anomalous name for this charm; we won't follow
+        # this pattern for future regions on amd64.
+        if (region, arch) == ("lcy02", "amd64"):
+            lbim_app_name = "launchpad-buildd-image-modifier-amd64"
+        else:
+            lbim_app_name = f"launchpad-buildd-image-modifier-{region}-{arch}"
+        upgrade_charm(lbim_app_name, "launchpad-buildd-image-modifier")
+        upgrade_charm(f"vbuilder-manage-{region}", "vbuilder-manage")
+
+
+if __name__ == "__main__":
+    main()