launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #31963
[Merge] ~tushar5526/lpbuildbot-worker:add-support-for-creating-base-image-flavors-for-testing into lpbuildbot-worker:main
Tushar Gupta has proposed merging ~tushar5526/lpbuildbot-worker:add-support-for-creating-base-image-flavors-for-testing into lpbuildbot-worker:main.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~tushar5526/lpbuildbot-worker/+git/lpbuildbot-worker/+merge/477795
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~tushar5526/lpbuildbot-worker:add-support-for-creating-base-image-flavors-for-testing into lpbuildbot-worker:main.
diff --git a/README.md b/README.md
index 559d71b..47355fa 100644
--- a/README.md
+++ b/README.md
@@ -96,3 +96,36 @@ SLAVE_PREFIXCMD[1]="" # prefix command, i.e. nice, linux32, dchr
### Run the worker
`sudo /etc/init.d/buildslave start`
+
+# Development Guide
+
+Create a new virtual environment:
+
+```
+python -m venv <path-to-virtual-env>
+source <path-to-virtual-env>/bin/activate
+```
+
+Install the required dependencies
+
+```
+pip install -r charm/requirements-dev.txt
+```
+
+### Running the tests
+
+You can run the tests using tox. Install `tox` and install the required python interpreters you want to test on using `pyenv`.
+
+Once you have the necessary python versions installed. Enable them globally so `tox` discovers them. For example, you can run:
+
+ ```
+ pyenv global 3.6 3.7 ..
+ ```
+
+Test files are present in `charms/tests`. Finally, run tests using tox.
+
+```
+cd charms
+tox
+tox -e py36 py37 # if you want to run test for specific versions
+```
diff --git a/charm/config.yaml b/charm/config.yaml
index abf0721..2a06651 100644
--- a/charm/config.yaml
+++ b/charm/config.yaml
@@ -7,6 +7,11 @@ options:
default: xenial
description: >
Space-separated list of Ubuntu series for which to maintain workers.
+ series-flavors:
+ type: string
+ default: ""
+ description: >
+ Space-separated list of {series}-{flavors} that needs to be maintained. Supported flavors are focal-postgres-14.
manager-host:
type: string
default:
diff --git a/charm/requirements.txt b/charm/requirements.txt
index e352ba9..755e629 100644
--- a/charm/requirements.txt
+++ b/charm/requirements.txt
@@ -1,2 +1,2 @@
jinja2
-ops >= 1.2.0
+ops==1.5.2
diff --git a/charm/src/charm.py b/charm/src/charm.py
index 2fe3960..763f252 100755
--- a/charm/src/charm.py
+++ b/charm/src/charm.py
@@ -28,6 +28,16 @@ class BlockedOnConfig(Exception):
super().__init__("Waiting for {} to be set".format(name))
+class InvalidFlavorValueFormat(Exception):
+ def __init__(self, flavor):
+ super().__init__(
+ "Invalid flavor format {}. Flavors should follow the \
+format (series)-(flavor).".format(
+ flavor
+ )
+ )
+
+
class LPBuildbotWorkerCharm(CharmBase):
_stored = StoredState()
@@ -191,6 +201,7 @@ class LPBuildbotWorkerCharm(CharmBase):
def _make_workers(self):
ubuntu_series = set(self._require_config("ubuntu-series").split())
+ series_flavors = set(self.config.get("series-flavors").split())
manager_host = self._require_config("manager-host")
manager_port = self._require_config("manager-port")
buildbot_password = self._require_config("buildbot-password")
@@ -200,42 +211,30 @@ class LPBuildbotWorkerCharm(CharmBase):
self._run_as_buildbot(
["mkdir", "-m755", "-p", str(workers_path)], check=True
)
-
current_images = self._list_lxd_images()
for series in ubuntu_series:
- self._set_maintenance_step("Making worker for {}".format(series))
- base_path = workers_path / "{}-lxd-worker".format(series)
- if not base_path.exists():
- self._run_as_buildbot(
- ["mkdir", "-m755", "-p", str(base_path)], check=True
- )
- lp_path = base_path / "devel"
- dependencies_path = base_path / "dependencies"
- download_cache_path = dependencies_path / "download-cache"
- if not lp_path.exists():
- self._run_as_buildbot(
- [
- "git",
- "clone",
- "https://git.launchpad.net/launchpad";,
- str(lp_path),
- ],
- check=True,
- )
- if not download_cache_path.exists():
- self._run_as_buildbot(
- [
- "git",
- "clone",
- "https://git.launchpad.net/lp-source-dependencies";,
- str(download_cache_path),
- ],
- check=True,
- )
- if "lptests-{}".format(series) not in current_images:
- self._run_as_buildbot(
- ["create-lp-tests-lxd", series, str(base_path)], check=True
+ base_path = self._setup_launchpad_working_dir(series, workers_path)
+ self._create_lxd_image(series, base_path, current_images)
+
+ for series_flavor in series_flavors:
+ try:
+ series, flavor = series_flavor.split("-", 1)
+ except ValueError:
+ raise InvalidFlavorValueFormat(series_flavor)
+
+ # build flavor for a series if the series is active
+ if series not in ubuntu_series:
+ logger.info(
+ "Skipping {} as series {} is not active".format(
+ series_flavor, series
+ )
)
+ continue
+
+ base_path = self._setup_launchpad_working_dir(
+ series, workers_path, flavor
+ )
+ self._create_lxd_image(series, base_path, current_images, flavor)
self._render_template(
"buildbot.tac.j2",
@@ -286,6 +285,53 @@ class LPBuildbotWorkerCharm(CharmBase):
self._stored.ubuntu_series = ubuntu_series
+ def _setup_launchpad_working_dir(self, series, workers_path, flavor=""):
+ flavor_suffix = "-" + flavor if flavor else ""
+ self._set_maintenance_step(
+ "Making worker for {}{}".format(series, flavor_suffix)
+ )
+ base_path = workers_path / "{}{}-lxd-worker".format(
+ series, flavor_suffix
+ )
+ if not base_path.exists():
+ self._run_as_buildbot(
+ ["mkdir", "-m755", "-p", str(base_path)], check=True
+ )
+ lp_path = base_path / "devel"
+ dependencies_path = base_path / "dependencies"
+ download_cache_path = dependencies_path / "download-cache"
+ if not lp_path.exists():
+ self._run_as_buildbot(
+ [
+ "git",
+ "clone",
+ "https://git.launchpad.net/launchpad";,
+ str(lp_path),
+ ],
+ check=True,
+ )
+ if not download_cache_path.exists():
+ self._run_as_buildbot(
+ [
+ "git",
+ "clone",
+ "https://git.launchpad.net/lp-source-dependencies";,
+ str(download_cache_path),
+ ],
+ check=True,
+ )
+ return base_path
+
+ def _create_lxd_image(self, series, base_path, current_images, flavor=""):
+ if (
+ "lptests-{}{}".format(series, "-" + flavor if flavor else "")
+ not in current_images
+ ):
+ self._run_as_buildbot(
+ ["create-lp-tests-lxd", series, str(base_path), flavor],
+ check=True,
+ )
+
def _on_install(self, event):
self._install_lpbuildbot_worker()
self._install_lxd()
diff --git a/charm/tests/test_charm.py b/charm/tests/test_charm.py
index f3efcd0..bc3ed1c 100644
--- a/charm/tests/test_charm.py
+++ b/charm/tests/test_charm.py
@@ -12,7 +12,11 @@ import pytest
from ops.model import ActiveStatus, MaintenanceStatus
from ops.testing import Harness
-from charm import BlockedOnConfig, LPBuildbotWorkerCharm
+from charm import (
+ BlockedOnConfig,
+ InvalidFlavorValueFormat,
+ LPBuildbotWorkerCharm,
+)
# The "fs" fixture is a fake filesystem from pyfakefs; the "fp" fixture is
# from pytest-subprocess.
@@ -224,7 +228,7 @@ def test_make_workers_requires_config(harness, fs, fp, empty_key):
pytest.raises(BlockedOnConfig, harness.charm._make_workers)
-def test_make_workers(harness, fs, fp, fake_user):
+def test_make_only_vanilla_ubuntu_workers(harness, fs, fp, fake_user):
fs.add_real_directory(harness.charm.charm_dir / "templates")
fp.keep_last_process(True)
fp.register(
@@ -287,6 +291,7 @@ def test_make_workers(harness, fs, fp, fake_user):
"create-lp-tests-lxd",
"xenial",
"/var/lib/buildbot/slaves/xenial-lxd-worker",
+ "",
],
["update-rc.d", "buildslave", "defaults"],
["service", "buildslave", "restart"],
@@ -303,6 +308,151 @@ def test_make_workers(harness, fs, fp, fake_user):
assert harness.charm._stored.ubuntu_series == {"xenial"}
+def test_make_both_vanilla_ubuntu_and_flavors_workers(
+ harness, fs, fp, fake_user
+):
+ fs.add_real_directory(harness.charm.charm_dir / "templates")
+ fp.keep_last_process(True)
+ fp.register(
+ ["sudo", "-Hu", "buildbot", "lxc", "image", "list", "-f", "json"],
+ stdout=json.dumps({}),
+ )
+ fp.register([fp.any()])
+ harness._update_config(
+ {
+ "manager-host": "manager.example.com",
+ "buildbot-password": "secret",
+ "ubuntu-series": "focal",
+ # focal-postgres14 should be allowed as focal
+ # series is enabled. xenial-postgres14 should be
+ # skipped
+ "series-flavors": "focal-postgres14 xenial-postgres14",
+ }
+ )
+
+ harness.charm._make_workers()
+
+ # we can only access the final status that was set
+ assert list(fp.calls) == [
+ [
+ "sudo",
+ "-Hu",
+ "buildbot",
+ "mkdir",
+ "-m755",
+ "-p",
+ "/var/lib/buildbot/slaves",
+ ],
+ ["sudo", "-Hu", "buildbot", "lxc", "image", "list", "-f", "json"],
+ [
+ "sudo",
+ "-Hu",
+ "buildbot",
+ "mkdir",
+ "-m755",
+ "-p",
+ "/var/lib/buildbot/slaves/focal-lxd-worker",
+ ],
+ [
+ "sudo",
+ "-Hu",
+ "buildbot",
+ "git",
+ "clone",
+ "https://git.launchpad.net/launchpad";,
+ "/var/lib/buildbot/slaves/focal-lxd-worker/devel",
+ ],
+ [
+ "sudo",
+ "-Hu",
+ "buildbot",
+ "git",
+ "clone",
+ "https://git.launchpad.net/lp-source-dependencies";,
+ "/var/lib/buildbot/slaves/focal-lxd-worker/"
+ "dependencies/download-cache",
+ ],
+ [
+ "sudo",
+ "-Hu",
+ "buildbot",
+ "create-lp-tests-lxd",
+ "focal",
+ "/var/lib/buildbot/slaves/focal-lxd-worker",
+ "",
+ ],
+ [
+ "sudo",
+ "-Hu",
+ "buildbot",
+ "mkdir",
+ "-m755",
+ "-p",
+ "/var/lib/buildbot/slaves/focal-postgres14-lxd-worker",
+ ],
+ [
+ "sudo",
+ "-Hu",
+ "buildbot",
+ "git",
+ "clone",
+ "https://git.launchpad.net/launchpad";,
+ "/var/lib/buildbot/slaves/focal-postgres14-lxd-worker/devel",
+ ],
+ [
+ "sudo",
+ "-Hu",
+ "buildbot",
+ "git",
+ "clone",
+ "https://git.launchpad.net/lp-source-dependencies";,
+ "/var/lib/buildbot/slaves/focal-postgres14-lxd-worker/"
+ "dependencies/download-cache",
+ ],
+ [
+ "sudo",
+ "-Hu",
+ "buildbot",
+ "create-lp-tests-lxd",
+ "focal",
+ "/var/lib/buildbot/slaves/focal-postgres14-lxd-worker",
+ "postgres14",
+ ],
+ ["update-rc.d", "buildslave", "defaults"],
+ ["service", "buildslave", "restart"],
+ ]
+ buildbot_tac = (
+ Path("/var/lib/buildbot/slaves/buildbot.tac").read_text().splitlines()
+ )
+ assert "basedir = '/var/lib/buildbot/slaves'" in buildbot_tac
+ assert "manager_host = 'manager.example.com'" in buildbot_tac
+ assert "manager_port = 9989" in buildbot_tac
+ assert "password = 'secret'" in buildbot_tac
+ buildslave = Path("/etc/default/buildslave").read_text()
+ assert 'SLAVE_BASEDIR[1]="/var/lib/buildbot/slaves"' in buildslave
+ assert harness.charm._stored.ubuntu_series == {"focal"}
+
+
+def test_raise_error_workers(harness, fs, fp, fake_user):
+ fs.add_real_directory(harness.charm.charm_dir / "templates")
+ fp.keep_last_process(True)
+ fp.register(
+ ["sudo", "-Hu", "buildbot", "lxc", "image", "list", "-f", "json"],
+ stdout=json.dumps({}),
+ )
+ fp.register([fp.any()])
+ harness._update_config(
+ {
+ "manager-host": "manager.example.com",
+ "buildbot-password": "secret",
+ "ubuntu-series": "focal",
+ "series-flavors": "postgres14",
+ }
+ )
+
+ pytest.raises(InvalidFlavorValueFormat, harness.charm._make_workers)
+
+
def test_make_workers_deletes_obsolete_workers(harness, fs, fp, fake_user):
fs.add_real_directory(harness.charm.charm_dir / "templates")
fp.keep_last_process(True)
diff --git a/create-lp-tests-lxd b/create-lp-tests-lxd
index fdd54fe..5ba9ba4 100755
--- a/create-lp-tests-lxd
+++ b/create-lp-tests-lxd
@@ -5,6 +5,7 @@ import os
import shlex
import subprocess
import sys
+from collections import defaultdict
from datetime import datetime
from os.path import expanduser
from pathlib import Path
@@ -57,6 +58,22 @@ LXD_HOSTS_CONTENT = (
)
+class Flavor:
+ def __init__(self, extra_ppas=None, extra_lp_deb_dependencies=None):
+ self.extra_ppas = list(extra_ppas) if extra_ppas else []
+ self.extra_lp_deb_dependencies = (
+ list(extra_lp_deb_dependencies)
+ if extra_lp_deb_dependencies
+ else []
+ )
+
+
+FLAVORS = defaultdict(Flavor)
+FLAVORS["postgres14"] = Flavor(
+ ["ppa:launchpad/postgresql-ports"], ["launchpad-database-dependencies-14"]
+)
+
+
def _put_file(container, source, destination):
with open(source) as source_file:
container.files.put(destination, source_file.read())
@@ -157,7 +174,7 @@ def create_new_instance(client, image_name, series, code_directory):
return container
-def install_code(container, series, directory):
+def install_code(container, series, directory, flavor_name=""):
print("Configuring apt")
_exec(container, ["apt-get", "update"])
_exec(container, ["apt-get", "upgrade", "-y"])
@@ -165,12 +182,31 @@ def install_code(container, series, directory):
container, ["apt-get", "install", "-y", "software-properties-common"]
)
+ if flavor_name and flavor_name not in FLAVORS.keys():
+ raise ValueError(
+ "Flavor {} is not supported. \
+ Please provide one of the supported flavors - {}".format(
+ flavor_name, ", ".join(FLAVORS.keys())
+ )
+ )
+
+ flavor = FLAVORS[flavor_name]
+
+ for ppa in flavor.extra_ppas:
+ _exec(container, ["add-apt-repository", "-y", "{}".format(ppa)])
+
for apt_repository in APT_REPOSITORIES:
repository = apt_repository.format(distro=series)
_exec(container, ["add-apt-repository", "-y", "{}".format(repository)])
_exec(container, ["apt-get", "update"])
+ print("Installing flavor apt dependencies")
+ _exec(
+ container,
+ ["apt-get", "install", "-y"] + flavor.extra_lp_deb_dependencies,
+ )
+
print("Installing Launchpad apt dependencies")
_exec(
container,
@@ -284,9 +320,17 @@ if __name__ == "__main__":
parser.add_argument(
"directory", type=str, help="Directory to mount code from."
)
-
+ parser.add_argument(
+ "flavor",
+ type=str,
+ help="Series flavor to build. Possible values: {}".format(
+ ", ".join(FLAVORS.keys())
+ ),
+ )
args = parser.parse_args()
- image_name = "lptests-{}".format(args.series)
+ image_name = "lptests-{}{}".format(
+ args.series, "-" + args.flavor if args.flavor else ""
+ )
code_directory = args.directory
if not Path(code_directory).exists():
@@ -309,6 +353,6 @@ if __name__ == "__main__":
container = create_new_instance(
client, image_name, args.series, code_directory
)
- install_code(container, args.series, code_directory)
+ install_code(container, args.series, code_directory, args.flavor)
publish_image(container, image_name)
remove_build_container(container)
Follow ups
