launchpad-reviewers team mailing list archive

[Tushar Gupta <mp+477795@xxxxxxxxxxxxxxxxxx>]

Tushar Gupta has proposed merging ~tushar5526/lpbuildbot-worker:add-support-for-creating-base-image-flavors-for-testing into lpbuildbot-worker:main.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~tushar5526/lpbuildbot-worker/+git/lpbuildbot-worker/+merge/477795
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~tushar5526/lpbuildbot-worker:add-support-for-creating-base-image-flavors-for-testing into lpbuildbot-worker:main.

diff --git a/charm/config.yaml b/charm/config.yaml
index abf0721..75e9180 100644
--- a/charm/config.yaml
+++ b/charm/config.yaml
@@ -7,6 +7,11 @@ options:
     default: xenial
     description: >
       Space-separated list of Ubuntu series for which to maintain workers.
+  series-flavors:
+    type: string
+    default: ""
+    description: >
+      Space-separated list of {series}-{flavors} that needs to be maintained. Supported flavors are focal-postgres-14.  
   manager-host:
     type: string
     default:
diff --git a/charm/src/charm.py b/charm/src/charm.py
index 2fe3960..d717c00 100755
--- a/charm/src/charm.py
+++ b/charm/src/charm.py
@@ -191,6 +191,7 @@ class LPBuildbotWorkerCharm(CharmBase):
 
     def _make_workers(self):
         ubuntu_series = set(self._require_config("ubuntu-series").split())
+        series_flavors = set(self._require_config("series-flavors").split())
         manager_host = self._require_config("manager-host")
         manager_port = self._require_config("manager-port")
         buildbot_password = self._require_config("buildbot-password")
@@ -203,39 +204,18 @@ class LPBuildbotWorkerCharm(CharmBase):
 
         current_images = self._list_lxd_images()
         for series in ubuntu_series:
-            self._set_maintenance_step("Making worker for {}".format(series))
-            base_path = workers_path / "{}-lxd-worker".format(series)
-            if not base_path.exists():
-                self._run_as_buildbot(
-                    ["mkdir", "-m755", "-p", str(base_path)], check=True
-                )
-            lp_path = base_path / "devel"
-            dependencies_path = base_path / "dependencies"
-            download_cache_path = dependencies_path / "download-cache"
-            if not lp_path.exists():
-                self._run_as_buildbot(
-                    [
-                        "git",
-                        "clone",
-                        "https://git.launchpad.net/launchpad";,
-                        str(lp_path),
-                    ],
-                    check=True,
-                )
-            if not download_cache_path.exists():
-                self._run_as_buildbot(
-                    [
-                        "git",
-                        "clone",
-                        "https://git.launchpad.net/lp-source-dependencies";,
-                        str(download_cache_path),
-                    ],
-                    check=True,
-                )
-            if "lptests-{}".format(series) not in current_images:
-                self._run_as_buildbot(
-                    ["create-lp-tests-lxd", series, str(base_path)], check=True
-                )
+            base_path = self._setup_launchpad_working_dir(series, workers_path)
+            self._create_lxd_image(series, base_path, current_images)
+
+        for seires_flavor in series_flavors:
+            series, flavor = seires_flavor.split('-', 1)
+            
+            # build flavor for a series if the series is active
+            if not series in ubuntu_series:
+                continue
+
+            base_path = self._setup_launchpad_working_dir(series, workers_path, flavor)
+            self._create_lxd_image(series, base_path, current_images, flavor)
 
         self._render_template(
             "buildbot.tac.j2",
@@ -286,6 +266,45 @@ class LPBuildbotWorkerCharm(CharmBase):
 
         self._stored.ubuntu_series = ubuntu_series
 
+    def _setup_launchpad_working_dir(self, series, workers_path, flavor = ""):
+            flavor_suffix = '-' + flavor if flavor else ""
+            self._set_maintenance_step("Making worker for {}{}".format(series, flavor_suffix))
+            base_path = workers_path / "{}{}-lxd-worker".format(series, flavor_suffix)
+            if not base_path.exists():
+                self._run_as_buildbot(
+                    ["mkdir", "-m755", "-p", str(base_path)], check=True
+                )
+            lp_path = base_path / "devel"
+            dependencies_path = base_path / "dependencies"
+            download_cache_path = dependencies_path / "download-cache"
+            if not lp_path.exists():
+                self._run_as_buildbot(
+                    [
+                        "git",
+                        "clone",
+                        "https://git.launchpad.net/launchpad";,
+                        str(lp_path),
+                    ],
+                    check=True,
+                )
+            if not download_cache_path.exists():
+                self._run_as_buildbot(
+                    [
+                        "git",
+                        "clone",
+                        "https://git.launchpad.net/lp-source-dependencies";,
+                        str(download_cache_path),
+                    ],
+                    check=True,
+                )
+            return base_path
+    
+    def _create_lxd_image(self, series, base_path, current_images, flavor = ""):
+        if "lptests-{}{}".format(series, '-' + flavor if flavor else "") not in current_images:
+            self._run_as_buildbot(
+                ["create-lp-tests-lxd", series, str(base_path), flavor], check=True
+            )
+
     def _on_install(self, event):
         self._install_lpbuildbot_worker()
         self._install_lxd()
diff --git a/create-lp-tests-lxd b/create-lp-tests-lxd
index fdd54fe..0d9a971 100755
--- a/create-lp-tests-lxd
+++ b/create-lp-tests-lxd
@@ -56,6 +56,15 @@ LXD_HOSTS_CONTENT = (
     ("127.0.0.99", "bazaar.launchpad.test"),
 )
 
+class Flavor:
+    def __init__(self, extra_ppas = None, extra_lp_deb_dependencies = None):
+        self.extra_ppas = list(extra_ppas) if extra_ppas else None
+        self.extra_lp_deb_dependencies = list(extra_lp_deb_dependencies) if extra_lp_deb_dependencies else None
+
+FLAVORS = {
+    "postgres14" : Flavor(["ppa:launchpad/postgresql-ports"], ["launchpad-database-dependencies-14"])
+}
+
 
 def _put_file(container, source, destination):
     with open(source) as source_file:
@@ -157,13 +166,22 @@ def create_new_instance(client, image_name, series, code_directory):
     return container
 
 
-def install_code(container, series, directory):
+def install_code(container, series, directory, flavor_name):
     print("Configuring apt")
     _exec(container, ["apt-get", "update"])
     _exec(container, ["apt-get", "upgrade", "-y"])
     _exec(
         container, ["apt-get", "install", "-y", "software-properties-common"]
     )
+    
+    if flavor_name not in FLAVORS.keys():
+        raise ValueError("Flavor {} is not supported. \
+            Please provide one of the supported flavors - {}".format(flavor_name, ", ".join(FLAVORS.keys())))
+
+    flavor = FLAVORS[flavor_name]
+
+    for ppa in flavor.extra_ppas:
+        _exec(container, ["add-apt-repository", "-y", "{}".format(ppa)])
 
     for apt_repository in APT_REPOSITORIES:
         repository = apt_repository.format(distro=series)
@@ -171,6 +189,12 @@ def install_code(container, series, directory):
 
     _exec(container, ["apt-get", "update"])
 
+    print("Installing flavor apt dependencies")
+    _exec(
+        container,
+        ["apt-get", "install", "-y"] + flavor.extra_lp_deb_dependencies,
+    )
+
     print("Installing Launchpad apt dependencies")
     _exec(
         container,
@@ -284,9 +308,11 @@ if __name__ == "__main__":
     parser.add_argument(
         "directory", type=str, help="Directory to mount code from."
     )
-
+    parser.add_argument(
+        "flavor", type=str, help="Series flavor to build. Possible flavors (postgres14, postgres16)"
+    )
     args = parser.parse_args()
-    image_name = "lptests-{}".format(args.series)
+    image_name = "lptests-{}{}".format(args.series, '-' + args.flavor if args.flavor else "")
 
     code_directory = args.directory
     if not Path(code_directory).exists():
@@ -309,6 +335,6 @@ if __name__ == "__main__":
     container = create_new_instance(
         client, image_name, args.series, code_directory
     )
-    install_code(container, args.series, code_directory)
+    install_code(container, args.series, code_directory, args.flavor)
     publish_image(container, image_name)
     remove_build_container(container)