launchpad-reviewers team mailing list archive

[Jürgen Gmach <mp+422750@xxxxxxxxxxxxxxxxxx>]

Jürgen Gmach has proposed merging ~jugmac00/launchpad-buildd:process-env-variable-and-apt-repository-information into launchpad-buildd:master.

Commit message:
Pass environment variable and apt repository data to the CI runner

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~jugmac00/launchpad-buildd/+git/launchpad-buildd/+merge/422750
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~jugmac00/launchpad-buildd:process-env-variable-and-apt-repository-information into launchpad-buildd:master.

diff --git a/.gitignore b/.gitignore
index 42f1548..c40f16f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,3 +14,4 @@ debian/*.debhelper
 debian/*.substvars
 .tox/
 docs/_build/
+venv/
diff --git a/debian/changelog b/debian/changelog
index e213691..1483257 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,8 +1,12 @@
 launchpad-buildd (213) UNRELEASED; urgency=medium
 
+  [ Colin Watson ]
   * Dynamically generate configuration file in lpbuildd.tests.harness.
   * Remove unnecessary "slave" component from some file names.
 
+  [ Jürgen Gmach ]
+  * Pass environment variable and apt repository data to the CI runner.
+
  -- Colin Watson <cjwatson@xxxxxxxxxx>  Wed, 20 Apr 2022 16:03:01 +0100
 
 launchpad-buildd (212) focal; urgency=medium
diff --git a/lpbuildd/ci.py b/lpbuildd/ci.py
index b2afd4e..dba4d8f 100644
--- a/lpbuildd/ci.py
+++ b/lpbuildd/ci.py
@@ -58,6 +58,8 @@ class CIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
         self.revocation_endpoint = extra_args.get("revocation_endpoint")
         self.proxy_service = None
         self.job_status = {}
+        self.apt_repositories = extra_args.get("apt_repositories")
+        self.environment_variables = extra_args.get("environment_variables")
 
         super().initiate(files, chroot, extra_args)
 
@@ -76,6 +78,11 @@ class CIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
             args.extend(["--git-repository", self.git_repository])
         if self.git_path is not None:
             args.extend(["--git-path", self.git_path])
+        if self.apt_repositories is not None:
+            args.extend(["--apt-repositories", self.apt_repositories])
+        if self.environment_variables is not None:
+            args.extend(
+                ["--environment-variables"], self.environment_variables)
         try:
             snap_store_proxy_url = self._builder._config.get(
                 "proxy", "snapstore")
diff --git a/lpbuildd/target/run_ci.py b/lpbuildd/target/run_ci.py
index 9c71904..7e80c0d 100644
--- a/lpbuildd/target/run_ci.py
+++ b/lpbuildd/target/run_ci.py
@@ -92,6 +92,20 @@ class RunCI(BuilderProxyOperationMixin, Operation):
         parser.add_argument("job_name", help="job name to run")
         parser.add_argument(
             "job_index", type=int, help="index within job name to run")
+        parser.add_argument(
+            "--environment_variables",
+            type=str,
+            action="append",
+            default=[],
+            help="environment variables to pass along",
+        )
+        parser.add_argument(
+            "--apt_repositories",
+            type=str,
+            action="append",
+            default=[],
+            help="apt repositories to pass along",
+        )
 
     def run_job(self):
         logger.info("Running job phase...")
@@ -101,9 +115,21 @@ class RunCI(BuilderProxyOperationMixin, Operation):
         output_path = os.path.join("/build", "output", job_id)
         self.backend.run(["mkdir", "-p", output_path])
         lpcraft_args = [
-            "lpcraft", "-v", "run-one", "--output-directory", output_path,
-            self.args.job_name, str(self.args.job_index),
-            ]
+            "lpcraft",
+            "-v",
+            "run-one",
+            "--output-directory",
+            output_path,
+            self.args.job_name,
+            str(self.args.job_index),
+        ]
+        if "environment_variables" in self.args and self.args.environment_variables:
+            d = dict(pair.split("=", maxsplit=1) for pair in self.args.environment_variables)
+            for key, value in d.items():
+                lpcraft_args.extend(["--set-env", "%s=%s" % (key, value)])
+        if "apt_repositories" in self.args and self.args.apt_repositories:
+            for repository in self.args.apt_repositories:
+                lpcraft_args.extend(["--apt-replace-repositories", repository])
         escaped_lpcraft_args = (
             " ".join(shell_escape(arg) for arg in lpcraft_args))
         tee_args = ["tee", "%s.log" % output_path]
diff --git a/lpbuildd/target/tests/test_run_ci.py b/lpbuildd/target/tests/test_run_ci.py
index 7c91efe..7bcbebc 100644
--- a/lpbuildd/target/tests/test_run_ci.py
+++ b/lpbuildd/target/tests/test_run_ci.py
@@ -348,6 +348,52 @@ class TestRunCI(TestCase):
                 ], cwd="/build/tree", **env),
             ]))
 
+    def test_run_job_with_environment_variables(self):
+        args = [
+            "run-ci",
+            "--backend=fake", "--series=focal", "--arch=amd64", "1",
+            "--environment_variables", "PIP_INDEX_URL=http://example";,
+            "--environment_variables", "SOME_PATH=/etc/some_path",
+            "test", "0",
+            ]
+        run_ci = parse_args(args=args).operation
+        run_ci.run_job()
+        self.assertThat(run_ci.backend.run.calls, MatchesListwise([
+            RanCommand(["mkdir", "-p", "/build/output/test:0"]),
+            RanBuildCommand([
+                "/bin/bash", "-o", "pipefail", "-c",
+                "lpcraft -v run-one --output-directory /build/output/test:0 "
+                "test 0 "
+                "--set-env PIP_INDEX_URL=http://example "
+                "--set-env SOME_PATH=/etc/some_path "
+                "2>&1 "
+                "| tee /build/output/test:0.log",
+                ], cwd="/build/tree"),
+            ]))
+
+    def test_run_job_with_apt_repositories(self):
+        args = [
+            "run-ci",
+            "--backend=fake", "--series=focal", "--arch=amd64", "1",
+            "--apt_repositories", "deb http://archive.ubuntu.com/ubuntu/ focal main restricted",
+            "--apt_repositories", "deb http://archive.ubuntu.com/ubuntu/ focal universe",
+            "test", "0",
+            ]
+        run_ci = parse_args(args=args).operation
+        run_ci.run_job()
+        self.assertThat(run_ci.backend.run.calls, MatchesListwise([
+            RanCommand(["mkdir", "-p", "/build/output/test:0"]),
+            RanBuildCommand([
+                "/bin/bash", "-o", "pipefail", "-c",
+                "lpcraft -v run-one --output-directory /build/output/test:0 "
+                "test 0 "
+                "--apt-replace-repositories 'deb http://archive.ubuntu.com/ubuntu/ focal main restricted' "
+                "--apt-replace-repositories 'deb http://archive.ubuntu.com/ubuntu/ focal universe' "
+                "2>&1 "
+                "| tee /build/output/test:0.log",
+                ], cwd="/build/tree"),
+            ]))
+
     def test_run_succeeds(self):
         args = [
             "run-ci",