launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #31431
[Merge] ~pelpsi/launchpad-buildd:enable-fetch-service-for-rock-builds into launchpad-buildd:master
Simone Pelosi has proposed merging ~pelpsi/launchpad-buildd:enable-fetch-service-for-rock-builds into launchpad-buildd:master.
Commit message:
Enable fetch service for rock builds
Install the fetch service proxy on rock builds.
The reason why we need to configure some extra environment variables
is that some tools will not accept the self-signed certificate
used by the fetch-service by default (like cargo), and some tools
need to behave in a specific way for the fetch-service to work (like go).
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~pelpsi/launchpad-buildd/+git/launchpad-buildd/+merge/473074
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~pelpsi/launchpad-buildd:enable-fetch-service-for-rock-builds into launchpad-buildd:master.
diff --git a/lpbuildd/rock.py b/lpbuildd/rock.py
index 3559feb..f80abf7 100644
--- a/lpbuildd/rock.py
+++ b/lpbuildd/rock.py
@@ -32,6 +32,7 @@ class RockBuildManager(BuildManagerProxyMixin, DebianBuildManager):
self.channels = extra_args.get("channels", {})
self.proxy_url = extra_args.get("proxy_url")
self.revocation_endpoint = extra_args.get("revocation_endpoint")
+ self.use_fetch_service = extra_args.get("use_fetch_service")
self.proxy_service = None
super().initiate(files, chroot, extra_args)
@@ -52,6 +53,14 @@ class RockBuildManager(BuildManagerProxyMixin, DebianBuildManager):
args.extend(["--git-path", self.git_path])
if self.build_path is not None:
args.extend(["--build-path", self.build_path])
+ if self.use_fetch_service:
+ args.append("--use_fetch_service")
+ args.extend(
+ [
+ "--fetch-service-mitm-certificate",
+ self.secrets["fetch_service_mitm_certificate"],
+ ]
+ )
args.append(self.name)
self.runTargetSubProcess("build-rock", *args)
diff --git a/lpbuildd/target/build_rock.py b/lpbuildd/target/build_rock.py
index 3110e78..f3fb867 100644
--- a/lpbuildd/target/build_rock.py
+++ b/lpbuildd/target/build_rock.py
@@ -1,5 +1,6 @@
import logging
import os
+import base64
from lpbuildd.target.backend import check_path_escape
from lpbuildd.target.build_snap import SnapChannelsAction
@@ -10,7 +11,7 @@ from lpbuildd.target.vcs import VCSOperationMixin
RETCODE_FAILURE_INSTALL = 200
RETCODE_FAILURE_BUILD = 201
-
+MITM_CERTIFICATE_PATH = "/usr/local/share/ca-certificates/local-ca.crt"
logger = logging.getLogger(__name__)
@@ -38,11 +39,64 @@ class BuildRock(
"--build-path", default=".", help="location of rock to build."
)
parser.add_argument("name", help="name of rock to build")
+ parser.add_argument(
+ "--use_fetch_service",
+ default=False,
+ action="store_true",
+ help="use the fetch service instead of the builder proxy",
+ )
+ parser.add_argument(
+ "--fetch-service-mitm-certificate",
+ type=str,
+ help="content of the ca certificate",
+ )
def __init__(self, args, parser):
super().__init__(args, parser)
self.buildd_path = os.path.join("/home/buildd", self.args.name)
+ def install_mitm_certificate(self):
+ """Install ca certificate for the fetch service
+
+ This is necessary so the fetch service can man-in-the-middle all
+ requests when fetching dependencies.
+ """
+ with self.backend.open(
+ MITM_CERTIFICATE_PATH, mode="wb"
+ ) as local_ca_cert:
+ # Certificate is passed as a Base64 encoded string.
+ # It's encoded using `base64 -w0` on the cert file.
+ decoded_certificate = base64.b64decode(
+ self.args.fetch_service_mitm_certificate.encode("ASCII")
+ )
+ local_ca_cert.write(decoded_certificate)
+ os.fchmod(local_ca_cert.fileno(), 0o644)
+ self.backend.run(["update-ca-certificates"])
+
+ def install_snapd_proxy(self, proxy_url):
+ """Install snapd proxy
+
+ This is necessary so the proxy can communicate properly
+ with snapcraft.
+ """
+ if proxy_url:
+ self.backend.run(
+ ["snap", "set", "system", f"proxy.http={proxy_url}"]
+ )
+ self.backend.run(
+ ["snap", "set", "system", f"proxy.https={proxy_url}"]
+ )
+
+ def build_rock_proxy_environment(self, env):
+ """Extend a command environment to include rockcraftproxy variables."""
+ env["CARGO_HTTP_CAINFO"] = MITM_CERTIFICATE_PATH
+ env["GOPROXY"] = "direct"
+ return env
+
+ def restart_snapd(self):
+ # This is required to pick up the certificate
+ self.backend.run(["systemctl", "restart", "snapd"])
+
def install(self):
logger.info("Running install phase")
deps = []
@@ -89,6 +143,18 @@ class BuildRock(
)
else:
self.backend.run(["snap", "install", "--classic", "rockcraft"])
+
+ if self.args.use_fetch_service:
+ # Deleting apt cache /var/lib/apt/lists before
+ # installing the fetch service
+ self.backend.run(
+ ["rm", "-rf", "/var/lib/apt/lists/*"]
+ )
+ self.install_mitm_certificate()
+ self.install_snapd_proxy(proxy_url=self.args.proxy_url)
+ self.backend.run(["apt-get", "-y", "update"])
+ self.restart_snapd()
+
# With classic confinement, the snap can access the whole system.
# We could build the rock in /build, but we are using /home/buildd
# for consistency with other build types.
@@ -98,6 +164,8 @@ class BuildRock(
"""Collect git or bzr branch."""
logger.info("Running repo phase...")
env = self.build_proxy_environment(proxy_url=self.args.proxy_url)
+ if self.args.use_fetch_service:
+ env = self.build_rock_proxy_environment(env)
self.vcs_fetch(self.args.name, cwd="/home/buildd", env=env)
self.vcs_update_status(self.buildd_path)
@@ -108,6 +176,8 @@ class BuildRock(
)
check_path_escape(self.buildd_path, build_context_path)
env = self.build_proxy_environment(proxy_url=self.args.proxy_url)
+ if self.args.use_fetch_service:
+ env = self.build_rock_proxy_environment(env)
args = ["rockcraft", "pack", "-v", "--destructive-mode"]
self.run_build_command(args, env=env, cwd=build_context_path)
diff --git a/lpbuildd/target/tests/test_build_rock.py b/lpbuildd/target/tests/test_build_rock.py
index 8f7682d..f2ea470 100644
--- a/lpbuildd/target/tests/test_build_rock.py
+++ b/lpbuildd/target/tests/test_build_rock.py
@@ -262,6 +262,179 @@ class TestBuildRock(TestCase):
],
)
+ def test_install_certificate(self):
+ args = [
+ "build-rock",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--git-repository",
+ "lp:foo",
+ "--proxy-url",
+ "http://proxy.example:3128/";,
+ "test-image",
+ "--use_fetch_service",
+ "--fetch-service-mitm-certificate",
+ # Base64 content_of_cert
+ "Y29udGVudF9vZl9jZXJ0",
+ ]
+ build_rock = parse_args(args=args).operation
+ build_rock.bin = "/builderbin"
+ self.useFixture(FakeFilesystem()).add("/builderbin")
+ os.mkdir("/builderbin")
+ with open("/builderbin/lpbuildd-git-proxy", "w") as proxy_script:
+ proxy_script.write("proxy script\n")
+ os.fchmod(proxy_script.fileno(), 0o755)
+ build_rock.install()
+ self.assertThat(
+ build_rock.backend.run.calls,
+ MatchesListwise(
+ [
+ RanAptGet(
+ "install",
+ "python3",
+ "socat",
+ "git",
+ "python3-pip",
+ "python3-setuptools",
+ ),
+ RanSnap("install", "--classic", "rockcraft"),
+ RanCommand(["rm", "-rf", "/var/lib/apt/lists/*"]),
+ RanCommand(["update-ca-certificates"]),
+ RanCommand(
+ [
+ "snap",
+ "set",
+ "system",
+ "proxy.http=http://proxy.example:3128/";,
+ ]
+ ),
+ RanCommand(
+ [
+ "snap",
+ "set",
+ "system",
+ "proxy.https=http://proxy.example:3128/";,
+ ]
+ ),
+ RanAptGet("update"),
+ RanCommand(
+ [
+ "systemctl",
+ "restart",
+ "snapd",
+ ]
+ ),
+ RanCommand(["mkdir", "-p", "/home/buildd"]),
+ ]
+ ),
+ )
+ self.assertEqual(
+ (b"proxy script\n", stat.S_IFREG | 0o755),
+ build_rock.backend.backend_fs["/usr/local/bin/lpbuildd-git-proxy"],
+ )
+ self.assertEqual(
+ (
+ b"[global]\n"
+ b"http-proxy-host = proxy.example\n"
+ b"http-proxy-port = 3128\n",
+ stat.S_IFREG | 0o644,
+ ),
+ build_rock.backend.backend_fs["/root/.subversion/servers"],
+ )
+ self.assertEqual(
+ (
+ b"content_of_cert",
+ stat.S_IFREG | 0o644,
+ ),
+ build_rock.backend.backend_fs[
+ "/usr/local/share/ca-certificates/local-ca.crt"
+ ],
+ )
+
+ def test_install_snapd_proxy(self):
+ args = [
+ "build-rock",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--git-repository",
+ "lp:foo",
+ "--proxy-url",
+ "http://proxy.example:3128/";,
+ "test-image",
+ "--use_fetch_service",
+ "--fetch-service-mitm-certificate",
+ # Base64 content_of_cert
+ "Y29udGVudF9vZl9jZXJ0",
+ ]
+ build_rock = parse_args(args=args).operation
+ build_rock.bin = "/builderbin"
+ self.useFixture(FakeFilesystem()).add("/builderbin")
+ os.mkdir("/builderbin")
+ with open("/builderbin/lpbuildd-git-proxy", "w") as proxy_script:
+ proxy_script.write("proxy script\n")
+ os.fchmod(proxy_script.fileno(), 0o755)
+ build_rock.install()
+ self.assertThat(
+ build_rock.backend.run.calls,
+ MatchesListwise(
+ [
+ RanAptGet(
+ "install",
+ "python3",
+ "socat",
+ "git",
+ "python3-pip",
+ "python3-setuptools",
+ ),
+ RanSnap("install", "--classic", "rockcraft"),
+ RanCommand(["rm", "-rf", "/var/lib/apt/lists/*"]),
+ RanCommand(["update-ca-certificates"]),
+ RanCommand(
+ [
+ "snap",
+ "set",
+ "system",
+ "proxy.http=http://proxy.example:3128/";,
+ ]
+ ),
+ RanCommand(
+ [
+ "snap",
+ "set",
+ "system",
+ "proxy.https=http://proxy.example:3128/";,
+ ]
+ ),
+ RanAptGet("update"),
+ RanCommand(
+ [
+ "systemctl",
+ "restart",
+ "snapd",
+ ]
+ ),
+ RanCommand(["mkdir", "-p", "/home/buildd"]),
+ ]
+ ),
+ )
+ self.assertEqual(
+ (b"proxy script\n", stat.S_IFREG | 0o755),
+ build_rock.backend.backend_fs["/usr/local/bin/lpbuildd-git-proxy"],
+ )
+ self.assertEqual(
+ (
+ b"[global]\n"
+ b"http-proxy-host = proxy.example\n"
+ b"http-proxy-port = 3128\n",
+ stat.S_IFREG | 0o644,
+ ),
+ build_rock.backend.backend_fs["/root/.subversion/servers"],
+ )
+
def test_repo_bzr(self):
args = [
"build-rock",
@@ -601,6 +774,46 @@ class TestBuildRock(TestCase):
),
)
+ def test_build_fetch_service(self):
+ args = [
+ "build-rock",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--branch",
+ "lp:foo",
+ "--proxy-url",
+ "http://proxy.example:3128/";,
+ "test-image",
+ "--use_fetch_service",
+ "--fetch-service-mitm-certificate",
+ # Base64 content_of_cert
+ "Y29udGVudF9vZl9jZXJ0",
+ ]
+ build_rock = parse_args(args=args).operation
+ build_rock.build()
+ env = {
+ "http_proxy": "http://proxy.example:3128/";,
+ "https_proxy": "http://proxy.example:3128/";,
+ "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy",
+ "SNAPPY_STORE_NO_CDN": "1",
+ 'CARGO_HTTP_CAINFO': '/usr/local/share/ca-certificates/local-ca.crt',
+ 'GOPROXY': 'direct',
+ }
+ self.assertThat(
+ build_rock.backend.run.calls,
+ MatchesListwise(
+ [
+ RanBuildCommand(
+ ["rockcraft", "pack", "-v", "--destructive-mode"],
+ cwd="/home/buildd/test-image/.",
+ **env,
+ ),
+ ]
+ ),
+ )
+
def test_run_succeeds(self):
args = [
"build-rock",
diff --git a/lpbuildd/tests/test_rock.py b/lpbuildd/tests/test_rock.py
index 5c56b5c..d4f8ad3 100644
--- a/lpbuildd/tests/test_rock.py
+++ b/lpbuildd/tests/test_rock.py
@@ -238,3 +238,18 @@ class TestRockBuildManagerIteration(TestCase):
self.buildmanager.iterate, self.buildmanager.iterators[-1]
)
self.assertFalse(self.builder.wasCalled("buildFail"))
+
+ @defer.inlineCallbacks
+ def test_iterate_use_fetch_service(self):
+ # The build manager can be told to use the fetch service as its proxy.
+ # This requires also a ca certificate passed in via secrets.
+ args = {
+ "use_fetch_service": True,
+ "secrets": {"fetch_service_mitm_certificate": "content_of_cert"},
+ }
+ expected_options = [
+ "--use_fetch_service",
+ "--fetch-service-mitm-certificate",
+ "content_of_cert",
+ ]
+ yield self.startBuild(args, expected_options)
