launchpad-reviewers team mailing list archive

[Jürgen Gmach <mp+426759@xxxxxxxxxxxxxxxxxx>]

Jürgen Gmach has proposed merging ~jugmac00/launchpad-buildd:process-secrets into launchpad-buildd:master.

Commit message:
Pass secrets to the CI runner

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~jugmac00/launchpad-buildd/+git/launchpad-buildd/+merge/426759
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~jugmac00/launchpad-buildd:process-secrets into launchpad-buildd:master.

diff --git a/debian/changelog b/debian/changelog
index 7a37ee9..34fdff6 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -8,6 +8,9 @@ launchpad-buildd (216) UNRELEASED; urgency=medium
   * Add a timeout when revoking proxy tokens.
   * Log SHA-512 hash of built snaps (LP: #1979844).
 
+  [ Jürgen Gmach ]
+  * Pass secrets to the CI runner.
+
  -- Andrey Fedoseev <andrey.fedoseev@xxxxxxxxxxxxx>  Mon, 27 Jun 2022 12:32:05 +0500
 
 launchpad-buildd (215) focal; urgency=medium
diff --git a/lpbuildd/ci.py b/lpbuildd/ci.py
index 3119a09..c420b9a 100644
--- a/lpbuildd/ci.py
+++ b/lpbuildd/ci.py
@@ -61,6 +61,7 @@ class CIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
         self.apt_repositories = extra_args.get("apt_repositories")
         self.environment_variables = extra_args.get("environment_variables")
         self.plugin_settings = extra_args.get("plugin_settings")
+        self.secrets = extra_args.get("secrets")
 
         super().initiate(files, chroot, extra_args)
 
@@ -149,6 +150,10 @@ class CIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
             for key, value in self.plugin_settings.items():
                 args.extend(
                     ["--plugin-setting", f"{key}={value}"])
+        if self.secrets is not None:
+            for key, value in self.secrets.items():
+                args.extend(
+                    ["--secrets", f"{key}={value}"])
         job_name, job_index = self.current_job
         self.current_job_id = _make_job_id(job_name, job_index)
         args.extend([job_name, str(job_index)])
diff --git a/lpbuildd/target/run_ci.py b/lpbuildd/target/run_ci.py
index d21204a..49d0e63 100644
--- a/lpbuildd/target/run_ci.py
+++ b/lpbuildd/target/run_ci.py
@@ -3,6 +3,10 @@
 
 import logging
 import os
+import tempfile
+from pathlib import Path
+
+import yaml
 
 from lpbuildd.target.build_snap import SnapChannelsAction
 from lpbuildd.target.operation import Operation
@@ -116,6 +120,11 @@ class RunCI(BuilderProxyOperationMixin, Operation):
             default=[],
             help="plugin setting where the key and value are separated by =",
         )
+        parser.add_argument(
+            "--secrets",
+            type=Path,
+            help="secrets provided in a YAML configuration file",
+        )
 
     def run_job(self):
         logger.info("Running job phase...")
@@ -149,6 +158,14 @@ class RunCI(BuilderProxyOperationMixin, Operation):
         )
         for key, value in plugin_settings.items():
             lpcraft_args.extend(["--plugin-setting", f"{key}={value}"])
+        if self.args.secrets:
+            text = yaml.dump(self.args.secrets)
+            with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
+                f.write(text)
+                path_to_secrets = f.name
+            self.backend.copy_in(
+                path_to_secrets, "/tmp/.launchpad-secrets.yaml")
+            lpcraft_args.extend(["--secrets", "/tmp/.launchpad-secrets.yaml"])
 
         escaped_lpcraft_args = (
             " ".join(shell_escape(arg) for arg in lpcraft_args))
diff --git a/lpbuildd/target/tests/test_run_ci.py b/lpbuildd/target/tests/test_run_ci.py
index f7c2231..1b27409 100644
--- a/lpbuildd/target/tests/test_run_ci.py
+++ b/lpbuildd/target/tests/test_run_ci.py
@@ -419,6 +419,27 @@ class TestRunCI(TestCase):
                 ], cwd="/build/tree"),
             ]))
 
+    def test_run_job_with_secrets(self):
+        args = [
+            "run-ci",
+            "--backend=fake", "--series=focal", "--arch=amd64", "1",
+            "--secrets", "path/to/tempfile",
+            "test", "0",
+            ]
+        run_ci = parse_args(args=args).operation
+        run_ci.run_job()
+        self.assertThat(run_ci.backend.run.calls, MatchesListwise([
+            RanCommand(["mkdir", "-p", "/build/output/test:0"]),
+            RanBuildCommand([
+                "/bin/bash", "-o", "pipefail", "-c",
+                "lpcraft -v run-one --output-directory /build/output/test:0 "
+                "test 0 "
+                "--secrets /tmp/.launchpad-secrets.yaml "
+                "2>&1 "
+                "| tee /build/output/test:0.log",
+                ], cwd="/build/tree"),
+            ]))
+
     def test_run_succeeds(self):
         args = [
             "run-ci",
diff --git a/lpbuildd/tests/test_ci.py b/lpbuildd/tests/test_ci.py
index 1582b92..4e17860 100644
--- a/lpbuildd/tests/test_ci.py
+++ b/lpbuildd/tests/test_ci.py
@@ -257,6 +257,9 @@ class TestCIBuildManagerIteration(TestCase):
                 "miniconda_conda_channel": "https://user:pass@xxxxxxxxxxxxxxxxxxxxx/artifactory/soss-conda-stable-local/";,  # noqa: E501
                 "foo": "bar",
             },
+            "secrets": {
+                "auth": "user:pass",
+            }
         }
         expected_prepare_options = [
             "--git-repository", "https://git.launchpad.test/~example/+git/ci";,
@@ -272,6 +275,7 @@ class TestCIBuildManagerIteration(TestCase):
             "--environment-variable", "PATH=foo",
             "--plugin-setting", "miniconda_conda_channel=https://user:pass@xxxxxxxxxxxxxxxxxxxxx/artifactory/soss-conda-stable-local/";,  # noqa: E501
             "--plugin-setting", "foo=bar",
+            "--secrets", "auth=user:pass"
             ]
         yield self.expectRunJob("lint", "0", options=expected_job_options)
         self.buildmanager.backend.add_file(
diff --git a/system-dependencies.txt b/system-dependencies.txt
index ba2d9af..f4d4bd4 100644
--- a/system-dependencies.txt
+++ b/system-dependencies.txt
@@ -18,4 +18,5 @@ python3-systemfixtures
 python3-testtools
 python3-twisted
 python3-txfixtures
+python3-yaml
 python3-zope.interface