sts-sponsors team mailing list archive

[Christian Grabowski <mp+438502@xxxxxxxxxxxxxxxxxx>]

Christian Grabowski has proposed merging ~cgrabowski/maas-ci/+git/system-tests:ha_postgres_system_tests into ~maas-committers/maas-ci/+git/system-tests:master.

Commit message:
specify floating ip parameters

add ha postgres tests

add new roles



Requested reviews:
  MAAS Committers (maas-committers)

For more details, see:
https://code.launchpad.net/~cgrabowski/maas-ci/+git/system-tests/+merge/438502
-- 
Your team MAAS Committers is requested to review the proposed merge of ~cgrabowski/maas-ci/+git/system-tests:ha_postgres_system_tests into ~maas-committers/maas-ci/+git/system-tests:master.

diff --git a/config.yaml.sample b/config.yaml.sample
index e656a27..6ed5ebf 100644
--- a/config.yaml.sample
+++ b/config.yaml.sample
@@ -121,3 +121,4 @@ o11y:
 ansible-playbooks:
     git-repo: https://github.com/maas/maas-ansible-playbook.git
     git-branch: main
+    floating-ip-network: lxdbr0
diff --git a/setup.py b/setup.py
index 21d75cd..3143a0a 100644
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,7 @@
 from setuptools import find_packages, setup
 
 install_requires = (
+    'netaddr',
     'paramiko',
     'pytest',
     'pytest-dependency',
@@ -8,6 +9,7 @@ install_requires = (
     'pytest-steps',
     'pyyaml',
     'retry',
+    'requests',
     'ruamel.yaml',
 )
 
diff --git a/systemtests/ansible.py b/systemtests/ansible.py
index 8db10f9..37e7f06 100644
--- a/systemtests/ansible.py
+++ b/systemtests/ansible.py
@@ -9,6 +9,7 @@ from logging import getLogger
 from subprocess import CalledProcessError
 from typing import TYPE_CHECKING, Any, Iterator, Optional
 
+import requests
 from retry import retry
 
 from .api import AuthenticatedAPIClient, UnauthenticatedMAASAPIClient
@@ -216,12 +217,20 @@ class AnsibleHost:
         )
 
     @property
-    def has_postgres_primary(self) -> bool:
-        return self.has_role("maas_postgres_primary")
+    def has_corosync(self) -> bool:
+        return self.has_role("maas_corosync")
 
     @property
-    def has_postgres_secondary(self) -> bool:
-        return self.has_role("maas_postgres_secondary")
+    def has_pacemaker(self) -> bool:
+        return self.has_role("maas_pacemaker")
+
+    @property
+    def has_postgres(self) -> bool:
+        return self.has_role("maas_postgres")
+
+    @property
+    def has_postgres_proxy(self) -> bool:
+        return self.has_role("maas_postgres_proxy")
 
     @property
     def has_proxy(self) -> bool:
@@ -240,12 +249,20 @@ class AnsibleHost:
         self._add_role("maas_region_controller", config)
         return self
 
-    def add_postgres_primary(self, config: dict[str, str] = {}) -> AnsibleHost:
-        self._add_role("maas_postgres_primary", config)
+    def add_corosync(self, config: dict[str, str] = {}) -> AnsibleHost:
+        self._add_role("maas_corosync", config)
+        return self
+
+    def add_pacemaker(self, config: dict[str, str] = {}) -> AnsibleHost:
+        self._add_role("maas_pacemaker", config)
+        return self
+
+    def add_postgres(self, config: dict[str, str] = {}) -> AnsibleHost:
+        self._add_role("maas_postgres", config)
         return self
 
-    def add_postgres_secondary(self, config: dict[str, str] = {}) -> AnsibleHost:
-        self._add_role("maas_postgres_secondary", config)
+    def add_postgres_proxy(self, config: dict[str, str] = {}) -> AnsibleHost:
+        self._add_role("maas_postgres_proxy", config)
         return self
 
     def add_proxy(self, config: dict[str, str] = {}) -> AnsibleHost:
@@ -262,15 +279,34 @@ class AnsibleHost:
         self._remove_role("maas_rack_controller")
         self._remove_role("maas_region_controller")
 
-    def remove_postgres_primary(self) -> None:
-        self._remove_role("maas_postgres_primaryr")
+    def remove_corosync(self) -> None:
+        self._remove_role("maas_corosync")
 
-    def remove_postgres_secondary(self) -> None:
-        self._remove_role("maas_postgres_secondary")
+    def remove_pacemaker(self) -> None:
+        self._remove_role("maas_pacemaker")
+
+    def remove_postgres(self) -> None:
+        self._remove_role("maas_postgres")
+
+    def remove_postgres_proxy(self) -> None:
+        self._remove_role("maas_postgres_proxy")
 
     def remove_proxy(self) -> None:
         self._remove_role("maas_proxy")
 
+    def check_postgres_status(self) -> int:
+        if not (self.has_postgres and self.has_pacemaker and self.has_corosync):
+            return False
+        resp = requests.get(f"http://{ self.ip }:23267")
+        # assert response status is a valid response
+        assert resp.status_code == 200 or resp.status_code == 204
+        return resp.status_code
+
+    @property
+    def is_postgres_primary(self) -> bool:
+        # 204 is the status returned by the primary
+        return self.check_postgres_status() == 204
+
 
 class AnsibleMain:
     use_timeout = True
@@ -284,12 +320,14 @@ class AnsibleMain:
         playbooks_repo: str,
         playbooks_branch: str,
         proxy_env: Optional[dict[str, str]],
+        floating_ip_net: Optional[str],
     ) -> None:
         self._lxd = lxd
         self.instance = instance
         self._playbooks_repo = playbooks_repo
         self._playbooks_branch = playbooks_branch
         self._proxy_env = proxy_env
+        self._floating_ip_net = floating_ip_net or "lxdbr0"
         self._hosts_file = self.instance.files["/home/ubuntu/hosts"]
 
         self._base_config = self._proxy_env if self._proxy_env else {}
@@ -470,8 +508,10 @@ class AnsibleMain:
 
     def create_hosts_file(self) -> None:
         inventory: dict[str, list[AnsibleHost]] = {
-            "maas_postgres_primary": [],
-            "maas_postgres_secondary": [],
+            "maas_corosync": [],
+            "maas_pacemaker": [],
+            "maas_postgres": [],
+            "maas_postgres_proxy": [],
             "maas_proxy": [],
             "maas_region_controller": [],
             "maas_rack_controller": [],
@@ -494,12 +534,11 @@ class AnsibleMain:
             inv.append(f"[{role}]")
             inv.extend([host.host_setup(self.config, role) for host in hosts])
             inv.append("")
-        if [True for key in inventory.keys() if "postgres" in key]:
+        if [True for key in inventory.keys() if "corosync" in key]:
             inv.extend(
                 [
-                    "[maas_postgres:children]",
-                    "maas_postgres_primary",
-                    "maas_postgres_secondary",
+                    "[maas_pacemaker:children]",
+                    "maas_corosync",
                     "",
                 ]
             )
@@ -530,6 +569,17 @@ class AnsibleMain:
         etc_ansible_cfg = self.instance.files["/etc/ansible/ansible.cfg"]
         etc_ansible_cfg.write(ansible_cfg.read())
 
+    def create_floating_ip(self) -> str:
+        subnet = self._lxd.get_subnet(self._floating_ip_net)
+        ip = str(subnet.next())
+        while ip in set(host.ip for host in self._inventory_):
+            ip = str(subnet.next())
+        return ip
+
+    def get_subnet_prefix_len(self) -> int:
+        subnet = self._lxd.get_subnet(self._floating_ip_net)
+        return subnet.prefixlen
+
     def run_playbook(self, playbook: str = "site.yaml", debug: str = "-v") -> None:
         self.create_hosts_file()
         cmd: list[str] = [
diff --git a/systemtests/ansible_tests/test_ansible.py b/systemtests/ansible_tests/test_ansible.py
index f06fc6c..aa66691 100644
--- a/systemtests/ansible_tests/test_ansible.py
+++ b/systemtests/ansible_tests/test_ansible.py
@@ -54,7 +54,7 @@ class TestAnsibleMAAS:
         self, installation_type: str, testlog: Logger, ansible_main: AnsibleMain
     ) -> None:
         ansible_main.logger = testlog
-        database = ansible_main.add_host(image="ubuntu:22.04").add_postgres_primary()
+        database = ansible_main.add_host(image="ubuntu:22.04").add_postgres()
         regionrack_host = ansible_main.add_host(image="ubuntu:22.04").add_region_rack()
         with ansible_main.collect_inventory():
             ansible_main.update_config(
@@ -78,7 +78,7 @@ class TestAnsibleMAAS:
         ansible_main.logger = testlog
         start_version = "3.1"
         upgrade_version = "3.3"
-        database = ansible_main.add_host().add_postgres_primary()
+        database = ansible_main.add_host().add_postgres()
         host = ansible_main.add_host().add_region_rack()
 
         with ansible_main.collect_inventory():
diff --git a/systemtests/ansible_tests/test_ha_postgres.py b/systemtests/ansible_tests/test_ha_postgres.py
new file mode 100644
index 0000000..5bcc748
--- /dev/null
+++ b/systemtests/ansible_tests/test_ha_postgres.py
@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from systemtests.ansible import AnsibleMain
+from systemtests.chaos import ChaosMixin
+
+if TYPE_CHECKING:
+    from logging import Logger
+
+
+DEFAULT_MAAS_VERSION = "3.3"
+
+
+@pytest.mark.skip_if_ansible_playbooks_unconfigured(
+    "Needs Ansible playbook configuration"
+)
+class TestAnsibleHAPostgres(ChaosMixin):
+    def test_primary_is_elected(
+        self, testlog: Logger, ansible_main: AnsibleMain
+    ) -> None:
+        ansible_main.logger = testlog
+        databases = [
+            ansible_main.add_host().add_postgres().add_pacemaker().add_corosync()
+            for _ in range(3)
+        ]
+        with ansible_main.collect_inventory():
+            floating_ip = ansible_main.create_floating_ip()
+            prefix_len = ansible_main.get_subnet_prefix_len()
+            ansible_main.update_config(
+                {
+                    "maas_postgres_password": "sekret",
+                    "maas_postgres_floating_ip": floating_ip,
+                    "maas_postgres_floating_ip_prefix_len": prefix_len,
+                }
+            )
+            ansible_main.run_playbook("site.yaml", "-vvv")
+            primary_found = False
+            for host in databases:
+                primary_found = primary_found or host.is_postgres_primary
+            assert primary_found
+
+    @pytest.mark.parametrize("installation_type", ["deb", "snap"])
+    def test_failover_to_new_primary(
+        self, installation_type: str, testlog: Logger, ansible_main: AnsibleMain
+    ) -> None:
+        ansible_main.logger = testlog
+        databases = [
+            ansible_main.add_host(image="ubuntu:22.04")
+            .add_postgres()
+            .add_pacemaker()
+            .add_corosync()
+            for _ in range(3)
+        ]
+        host = (
+            ansible_main.add_host(image="ubuntu:22.04")
+            .add_region()
+            .add_postgres_proxy()
+        )
+
+        with ansible_main.collect_inventory():
+            floating_ip = ansible_main.create_floating_ip()
+            prefix_len = ansible_main.get_subnet_prefix_len()
+            ansible_main.update_config(
+                {
+                    "maas_installation_type": installation_type,
+                    "maas_postgres_password": "sekret",
+                    "maas_url": f"http://{host.ip}:5240/MAAS";,
+                    "maas_version": DEFAULT_MAAS_VERSION,
+                    "maas_postgres_floating_ip": floating_ip,
+                    "maas_postgres_floating_ip_prefix_len": prefix_len,
+                }
+            )
+            ansible_main.run_playbook("site.yaml", "-vvv")
+            region = ansible_main.fetch_region(host)
+            assert (
+                region.read_version_information()["version"][:3] == DEFAULT_MAAS_VERSION
+            )
+            self.down_postgres_primary(ansible_main.lxd, databases)
diff --git a/systemtests/chaos.py b/systemtests/chaos.py
new file mode 100644
index 0000000..a3a8f1c
--- /dev/null
+++ b/systemtests/chaos.py
@@ -0,0 +1,21 @@
+import random
+from typing import TYPE_CHECKING, List
+
+from .ansible import AnsibleHost
+
+if TYPE_CHECKING:
+    from .lxd import CLILXD
+
+
+class ChaosMixin:
+    def down_a_host(self, lxd: CLILXD, hostname: str):
+        lxd.stop_instance(hostname)
+
+    def down_random_host(self, lxd: CLILXD, hostnames: List[str]):
+        target_host = random.randchoice(hostnames)
+        self.down_a_host(lxd, target_host)
+
+    def down_postgres_primary(self, lxd: CLILXD, ansible_hosts: List[AnsibleHost]):
+        for host in ansible_hosts:
+            if host.is_postgres_primary:
+                self.down_a_host(host.name)
diff --git a/systemtests/fixtures.py b/systemtests/fixtures.py
index decff3b..c2b0acd 100644
--- a/systemtests/fixtures.py
+++ b/systemtests/fixtures.py
@@ -45,6 +45,7 @@ def ansible_main(config: dict[str, Any]) -> Optional[Iterator[AnsibleMain]]:
     if not playbooks_repo:
         yield None
         return
+    floating_ip_net = playbooks_config.get("floating-ip-network")
     playbooks_branch = playbooks_config.get("git-branch", "main")
     log = getLogger(f"{LOG_NAME}.ansible_main")
     lxd = get_lxd(log)
@@ -57,6 +58,7 @@ def ansible_main(config: dict[str, Any]) -> Optional[Iterator[AnsibleMain]]:
         playbooks_repo=playbooks_repo,
         playbooks_branch=playbooks_branch,
         proxy_env=proxy_env,
+        floating_ip_net=floating_ip_net,
     )
     main.setup()
     yield main
diff --git a/systemtests/lxd.py b/systemtests/lxd.py
index cd91678..3592f41 100644
--- a/systemtests/lxd.py
+++ b/systemtests/lxd.py
@@ -8,6 +8,7 @@ from itertools import chain
 from pathlib import Path
 from typing import TYPE_CHECKING, Optional
 
+from netaddr import IPNetwork
 from retry import retry
 
 from .subprocess import run_with_logging
@@ -464,6 +465,12 @@ class CLILXD:
         )
         return Instance(self, instance_name)
 
+    def get_subnet(self, net_name, ip_version=4):
+        result = self._run(
+            ["lxc", "network", "get", net_name, f"ipv{ip_version}.address"]
+        )
+        return IPNetwork(result.stdout)
+
 
 def fmt_lxd_options(cfg: DeviceConfig) -> list[str]:
     exclude_options = ["device_name", "type"]