sts-sponsors team mailing list archive

[Adam Collard <mp+437129@xxxxxxxxxxxxxxxxxx>]

Adam Collard has proposed merging ~adam-collard/maas-ci/+git/system-tests:proxy-env-for-ansible into ~maas-committers/maas-ci/+git/system-tests:master.

Commit message:
[ansible-tests] Add proxy support



Requested reviews:
  MAAS Lander (maas-lander): unittests
  MAAS Committers (maas-committers)

For more details, see:
https://code.launchpad.net/~adam-collard/maas-ci/+git/system-tests/+merge/437129
-- 
Your team MAAS Committers is requested to review the proposed merge of ~adam-collard/maas-ci/+git/system-tests:proxy-env-for-ansible into ~maas-committers/maas-ci/+git/system-tests:master.

diff --git a/systemtests/ansible.py b/systemtests/ansible.py
index 2ca5b40..bc9265d 100644
--- a/systemtests/ansible.py
+++ b/systemtests/ansible.py
@@ -7,7 +7,7 @@ from datetime import timedelta
 from functools import cached_property
 from logging import getLogger
 from subprocess import CalledProcessError
-from typing import TYPE_CHECKING, Any, Iterator
+from typing import TYPE_CHECKING, Any, Iterator, Optional
 
 from retry import retry
 
@@ -56,23 +56,31 @@ def add_ansible_header(headers: list[str], config: dict[str, Any]) -> None:
         headers.append("ansible-playbooks: true")
 
 
-def apt_update(instance: Instance) -> None:
+def apt_update(
+    instance: Instance, environment: Optional[dict[str, str]] = None
+) -> None:
     """Update APT indices, fix broken dpkg."""
-    instance.quietly_execute(["apt-get", "update", "-y"])
-    instance.quietly_execute(["dpkg", "--configure", "-a"])
+    instance.quietly_execute(["apt-get", "update", "-y"], environment=environment)
+    instance.quietly_execute(["dpkg", "--configure", "-a"], environment=environment)
 
 
-def apt_install(instance: Instance, package: str) -> None:
+def apt_install(
+    instance: Instance, package: str, environment: Optional[dict[str, str]] = None
+) -> None:
     """Install given package from apt."""
-    instance.quietly_execute(["apt", "install", package, "-y"])
+    instance.quietly_execute(["apt", "install", package, "-y"], environment=environment)
 
 
-def pip_install(instance: Instance, package: str) -> None:
+def pip_install(
+    instance: Instance, package: str, environment: Optional[dict[str, str]] = None
+) -> None:
     """Ensure latest version of Python package is installed."""
     if not pip_package_exists(instance, package):
-        instance.quietly_execute(["pip3", "install", package])
+        instance.quietly_execute(["pip3", "install", package], environment=environment)
     else:
-        instance.quietly_execute(["pip3", "install", package, "--upgrade"])
+        instance.quietly_execute(
+            ["pip3", "install", package, "--upgrade"], environment=environment
+        )
 
 
 def pip_package_exists(instance: Instance, package: str) -> bool:
@@ -84,7 +92,13 @@ def pip_package_exists(instance: Instance, package: str) -> bool:
         return True
 
 
-def clone_repo(instance: Instance, repo: str, branch: str, clone_path: str) -> None:
+def clone_repo(
+    instance: Instance,
+    repo: str,
+    branch: str,
+    clone_path: str,
+    environment: Optional[dict[str, str]] = None,
+) -> None:
     clone_file = instance.files[clone_path]
     if not clone_file.exists():
         instance.execute(
@@ -98,6 +112,7 @@ def clone_repo(instance: Instance, repo: str, branch: str, clone_path: str) -> N
                 repo,
                 clone_path,
             ],
+            environment=environment,
         )
         instance.logger.info(f"Cloned {branch} from {repo} to {clone_path}")
 
@@ -265,11 +280,13 @@ class AnsibleMain:
         instance: Instance,
         playbooks_repo: str,
         playbooks_branch: str,
+        proxy_env: Optional[dict[str, str]],
     ) -> None:
         self._lxd = lxd
         self.instance = instance
         self._playbooks_repo = playbooks_repo
         self._playbooks_branch = playbooks_branch
+        self._proxy_env = proxy_env
         self._hosts_file = self.instance.files["/home/ubuntu/hosts"]
 
         self.config: dict[str, str] = {}
@@ -279,17 +296,18 @@ class AnsibleMain:
 
     def setup(self) -> None:
         self.logger.info("Installing python3-pip")
-        apt_update(self.instance)
-        apt_install(self.instance, "python3-pip")
+        apt_update(self.instance, environment=self._proxy_env)
+        apt_install(self.instance, "python3-pip", environment=self._proxy_env)
         self.logger.info("Installing ansible")
-        pip_install(self.instance, "ansible")
+        pip_install(self.instance, "ansible", environment=self._proxy_env)
         self.logger.info("Installing netaddr")
-        pip_install(self.instance, "netaddr")
+        pip_install(self.instance, "netaddr", environment=self._proxy_env)
         clone_repo(
             self.instance,
             self._playbooks_repo,
             self._playbooks_branch,
             self.ansible_repo_path,
+            environment=self._proxy_env,
         )
 
         self.create_config_file()
diff --git a/systemtests/fixtures.py b/systemtests/fixtures.py
index e552fed..1052736 100644
--- a/systemtests/fixtures.py
+++ b/systemtests/fixtures.py
@@ -50,11 +50,13 @@ def ansible_main(config: dict[str, Any]) -> Optional[Iterator[AnsibleMain]]:
     lxd = get_lxd(log)
     instance = Instance(lxd, "ansible-main")
     instance.create_container(config["containers-image"])
+    proxy_env = get_proxy_env(config.get("proxy", {}))
     main = AnsibleMain(
         lxd,
         instance,
         playbooks_repo=playbooks_repo,
         playbooks_branch=playbooks_branch,
+        proxy_env=proxy_env,
     )
     main.setup()
     yield main
@@ -110,6 +112,24 @@ def build_container(config: dict[str, Any]) -> Optional[Iterator[Instance]]:
     yield instance
 
 
+# TODO: Move to property of LXD, and expose on Instance
+def get_proxy_env(
+    proxy_config: dict[str, str], no_proxy: Optional[str] = None
+) -> Optional[dict[str, str]]:
+    """Get proxy related environment variables from config.yaml"""
+    http_proxy = proxy_config.get("http", "")
+    if http_proxy:
+        proxy_env = {
+            "http_proxy": http_proxy,
+            "https_proxy": http_proxy,
+        }
+        if no_proxy:
+            proxy_env["no_proxy"] = no_proxy
+        return proxy_env
+    else:
+        return None
+
+
 @pytest.fixture(scope="session")
 def maas_deb_repo(
     build_container: Optional[Instance], config: dict[str, Any]
@@ -120,17 +140,7 @@ def maas_deb_repo(
     else:
         build_container.logger = getLogger(f"{LOG_NAME}.maas_deb_repo")
         build_ip = build_container.get_ip_address()
-        http_proxy = config.get("proxy", {}).get("http", "")
-        proxy_env: Optional[dict[str, str]]
-        if http_proxy:
-            proxy_env = {
-                "http_proxy": http_proxy,
-                "https_proxy": http_proxy,
-                "no_proxy": build_ip,
-            }
-        else:
-            proxy_env = None
-
+        proxy_env = get_proxy_env(config.get("proxy", {}), no_proxy=build_ip)
         if not build_container.files["/var/www/html/repo/Packages.gz"].exists():
             maas_ppas = config.get("deb", {}).get(
                 "ppa", ["ppa:maas-committers/latest-deps"]