launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #31679
[Merge] ~pelpsi/launchpad-buildd:fetch-service-implementation-for-sourcecraft into launchpad-buildd:master
Simone Pelosi has proposed merging ~pelpsi/launchpad-buildd:fetch-service-implementation-for-sourcecraft into launchpad-buildd:master.
Commit message:
Fetch service configuration for Sourcecraft builds
Integrate use_fetch_service flag to activate or deactivate fetch service
for a given sourcecraft recipe.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~pelpsi/launchpad-buildd/+git/launchpad-buildd/+merge/474688
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~pelpsi/launchpad-buildd:fetch-service-implementation-for-sourcecraft into launchpad-buildd:master.
diff --git a/debian/changelog b/debian/changelog
index eb2ce28..075bcf6 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
launchpad-buildd (245) UNRELEASED; urgency=medium
* Shallow git clone when fetch-service enabled.
+ * Enable fetch service for sourcecraft builds.
-- Simone Pelosi <simone.pelosi@xxxxxxxxxxxxx> Tue, 08 Oct 2024 14:09:25 +0200
diff --git a/lpbuildd/source.py b/lpbuildd/source.py
index 3930f8f..1d7e890 100644
--- a/lpbuildd/source.py
+++ b/lpbuildd/source.py
@@ -32,6 +32,9 @@ class SourceBuildManager(BuildManagerProxyMixin, DebianBuildManager):
self.channels = extra_args.get("channels", {})
self.proxy_url = extra_args.get("proxy_url")
self.revocation_endpoint = extra_args.get("revocation_endpoint")
+ # currently only used to transport the mitm certificate
+ self.secrets = extra_args.get("secrets")
+ self.use_fetch_service = extra_args.get("use_fetch_service")
self.proxy_service = None
super().initiate(files, chroot, extra_args)
@@ -52,6 +55,14 @@ class SourceBuildManager(BuildManagerProxyMixin, DebianBuildManager):
args.extend(["--git-path", self.git_path])
if self.build_path is not None:
args.extend(["--build-path", self.build_path])
+ if self.use_fetch_service:
+ args.append("--use_fetch_service")
+ args.extend(
+ [
+ "--fetch-service-mitm-certificate",
+ self.secrets["fetch_service_mitm_certificate"],
+ ]
+ )
args.append(self.name)
self.runTargetSubProcess("build-source", *args)
diff --git a/lpbuildd/target/build_source.py b/lpbuildd/target/build_source.py
index 58eaece..9f866f9 100644
--- a/lpbuildd/target/build_source.py
+++ b/lpbuildd/target/build_source.py
@@ -11,6 +11,7 @@ from lpbuildd.target.vcs import VCSOperationMixin
RETCODE_FAILURE_INSTALL = 200
RETCODE_FAILURE_BUILD = 201
+MITM_CERTIFICATE_PATH = "/usr/local/share/ca-certificates/local-ca.crt"
logger = logging.getLogger(__name__)
@@ -38,6 +39,17 @@ class BuildSource(
"--build-path", default=".", help="location of source to build."
)
parser.add_argument("name", help="name of source to build")
+ parser.add_argument(
+ "--use_fetch_service",
+ default=False,
+ action="store_true",
+ help="use the fetch service instead of the builder proxy",
+ )
+ parser.add_argument(
+ "--fetch-service-mitm-certificate",
+ type=str,
+ help="content of the ca certificate",
+ )
def __init__(self, args, parser):
super().__init__(args, parser)
@@ -91,6 +103,16 @@ class BuildSource(
"sourcecraft",
]
)
+ if self.args.use_fetch_service:
+ # Deleting apt cache /var/lib/apt/lists before
+ # installing the fetch service
+ self.install_apt_proxy()
+ self.delete_apt_cache()
+ self.install_mitm_certificate()
+ self.install_snapd_proxy(proxy_url=self.args.proxy_url)
+ self.backend.run(["apt-get", "-y", "update"])
+ self.restart_snapd()
+ self.configure_git_protocol_v2()
# With classic confinement, the snap can access the whole system.
# We could build the source in /build, but we are using /home/buildd
# for consistency with other build types.
@@ -99,8 +121,18 @@ class BuildSource(
def repo(self):
"""Collect git or bzr branch."""
logger.info("Running repo phase...")
- env = self.build_proxy_environment(proxy_url=self.args.proxy_url)
- self.vcs_fetch(self.args.name, cwd="/home/buildd", env=env)
+ env = self.build_proxy_environment(
+ proxy_url=self.args.proxy_url,
+ use_fetch_service=self.args.use_fetch_service,
+ )
+ # using the fetch service requires shallow clones
+ git_shallow_clone = bool(self.args.use_fetch_service)
+ self.vcs_fetch(
+ self.args.name,
+ cwd="/home/buildd",
+ env=env,
+ git_shallow_clone_with_single_branch=git_shallow_clone,
+ )
self.vcs_update_status(self.buildd_path)
def build(self):
@@ -109,7 +141,10 @@ class BuildSource(
"/home/buildd", self.args.name, self.args.build_path
)
check_path_escape(self.buildd_path, build_context_path)
- env = self.build_proxy_environment(proxy_url=self.args.proxy_url)
+ env = self.build_proxy_environment(
+ proxy_url=self.args.proxy_url,
+ use_fetch_service=self.args.use_fetch_service,
+ )
args = ["sourcecraft", "pack", "-v", "--destructive-mode"]
self.run_build_command(args, env=env, cwd=build_context_path)
diff --git a/lpbuildd/target/tests/test_build_source.py b/lpbuildd/target/tests/test_build_source.py
index afbd43e..1d31860 100644
--- a/lpbuildd/target/tests/test_build_source.py
+++ b/lpbuildd/target/tests/test_build_source.py
@@ -7,7 +7,7 @@ from textwrap import dedent
import responses
from fixtures import FakeLogger, TempDir
from systemfixtures import FakeFilesystem
-from testtools.matchers import AnyMatch, MatchesAll, MatchesListwise
+from testtools.matchers import AnyMatch, MatchesAll, MatchesListwise, Not
from testtools.testcase import TestCase
from lpbuildd.target.backend import InvalidBuildFilePath
@@ -272,6 +272,277 @@ class TestBuildSource(TestCase):
],
)
+ def test_install_certificate(self):
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--git-repository",
+ "lp:foo",
+ "--proxy-url",
+ "http://proxy.example:3128/";,
+ "test-image",
+ "--use_fetch_service",
+ "--fetch-service-mitm-certificate",
+ # Base64 content_of_cert
+ "Y29udGVudF9vZl9jZXJ0",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.bin = "/builderbin"
+ self.useFixture(FakeFilesystem()).add("/builderbin")
+ os.mkdir("/builderbin")
+ with open("/builderbin/lpbuildd-git-proxy", "w") as proxy_script:
+ proxy_script.write("proxy script\n")
+ os.fchmod(proxy_script.fileno(), 0o755)
+ build_source.install()
+ self.assertThat(
+ build_source.backend.run.calls,
+ MatchesListwise(
+ [
+ RanAptGet(
+ "install",
+ "python3",
+ "socat",
+ "git",
+ ),
+ RanSnap(
+ "install",
+ "--classic",
+ "--channel=latest/edge/craftctl",
+ "sourcecraft",
+ ),
+ RanCommand(["rm", "-rf", "/var/lib/apt/lists"]),
+ RanCommand(["update-ca-certificates"]),
+ RanCommand(
+ [
+ "snap",
+ "set",
+ "system",
+ "proxy.http=http://proxy.example:3128/";,
+ ]
+ ),
+ RanCommand(
+ [
+ "snap",
+ "set",
+ "system",
+ "proxy.https=http://proxy.example:3128/";,
+ ]
+ ),
+ RanAptGet("update"),
+ RanCommand(
+ [
+ "systemctl",
+ "restart",
+ "snapd",
+ ]
+ ),
+ RanCommand(["mkdir", "-p", "/home/buildd"]),
+ ]
+ ),
+ )
+ self.assertEqual(
+ (b"proxy script\n", stat.S_IFREG | 0o755),
+ build_source.backend.backend_fs[
+ "/usr/local/bin/lpbuildd-git-proxy"
+ ],
+ )
+ self.assertEqual(
+ (
+ b"content_of_cert",
+ stat.S_IFREG | 0o644,
+ ),
+ build_source.backend.backend_fs[
+ "/usr/local/share/ca-certificates/local-ca.crt"
+ ],
+ )
+ self.assertEqual(
+ (
+ dedent(
+ """\
+ Acquire::http::Proxy "http://proxy.example:3128/";;
+ Acquire::https::Proxy "http://proxy.example:3128/";;
+
+ """
+ ).encode("UTF-8"),
+ stat.S_IFREG | 0o644,
+ ),
+ build_source.backend.backend_fs["/etc/apt/apt.conf.d/99proxy"],
+ )
+
+ def test_install_snapd_proxy(self):
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--git-repository",
+ "lp:foo",
+ "--proxy-url",
+ "http://proxy.example:3128/";,
+ "test-image",
+ "--use_fetch_service",
+ "--fetch-service-mitm-certificate",
+ # Base64 content_of_cert
+ "Y29udGVudF9vZl9jZXJ0",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.bin = "/builderbin"
+ self.useFixture(FakeFilesystem()).add("/builderbin")
+ os.mkdir("/builderbin")
+ with open("/builderbin/lpbuildd-git-proxy", "w") as proxy_script:
+ proxy_script.write("proxy script\n")
+ os.fchmod(proxy_script.fileno(), 0o755)
+ build_source.install()
+ self.assertThat(
+ build_source.backend.run.calls,
+ MatchesListwise(
+ [
+ RanAptGet(
+ "install",
+ "python3",
+ "socat",
+ "git",
+ ),
+ RanSnap(
+ "install",
+ "--classic",
+ "--channel=latest/edge/craftctl",
+ "sourcecraft",
+ ),
+ RanCommand(["rm", "-rf", "/var/lib/apt/lists"]),
+ RanCommand(["update-ca-certificates"]),
+ RanCommand(
+ [
+ "snap",
+ "set",
+ "system",
+ "proxy.http=http://proxy.example:3128/";,
+ ]
+ ),
+ RanCommand(
+ [
+ "snap",
+ "set",
+ "system",
+ "proxy.https=http://proxy.example:3128/";,
+ ]
+ ),
+ RanAptGet("update"),
+ RanCommand(
+ [
+ "systemctl",
+ "restart",
+ "snapd",
+ ]
+ ),
+ RanCommand(["mkdir", "-p", "/home/buildd"]),
+ ]
+ ),
+ )
+ self.assertEqual(
+ (b"proxy script\n", stat.S_IFREG | 0o755),
+ build_source.backend.backend_fs[
+ "/usr/local/bin/lpbuildd-git-proxy"
+ ],
+ )
+ self.assertEqual(
+ (
+ dedent(
+ """\
+ Acquire::http::Proxy "http://proxy.example:3128/";;
+ Acquire::https::Proxy "http://proxy.example:3128/";;
+
+ """
+ ).encode("UTF-8"),
+ stat.S_IFREG | 0o644,
+ ),
+ build_source.backend.backend_fs["/etc/apt/apt.conf.d/99proxy"],
+ )
+
+ def test_install_fetch_service(self):
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--git-repository",
+ "lp:foo",
+ "--proxy-url",
+ "http://proxy.example:3128/";,
+ "test-image",
+ "--use_fetch_service",
+ "--fetch-service-mitm-certificate",
+ # Base64 content_of_cert
+ "Y29udGVudF9vZl9jZXJ0",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.bin = "/builderbin"
+ self.useFixture(FakeFilesystem()).add("/builderbin")
+ os.mkdir("/builderbin")
+ with open("/builderbin/lpbuildd-git-proxy", "w") as proxy_script:
+ proxy_script.write("proxy script\n")
+ os.fchmod(proxy_script.fileno(), 0o755)
+ build_source.install()
+ self.assertThat(
+ build_source.backend.run.calls,
+ MatchesAll(
+ Not(
+ AnyMatch(
+ RanCommand(
+ [
+ "git",
+ "config",
+ "--global",
+ "protocol.version",
+ "2",
+ ]
+ )
+ )
+ ),
+ ),
+ )
+
+ def test_install_fetch_service_focal(self):
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=focal",
+ "--arch=amd64",
+ "1",
+ "--git-repository",
+ "lp:foo",
+ "--proxy-url",
+ "http://proxy.example:3128/";,
+ "test-image",
+ "--use_fetch_service",
+ "--fetch-service-mitm-certificate",
+ # Base64 content_of_cert
+ "Y29udGVudF9vZl9jZXJ0",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.bin = "/builderbin"
+ self.useFixture(FakeFilesystem()).add("/builderbin")
+ os.mkdir("/builderbin")
+ with open("/builderbin/lpbuildd-git-proxy", "w") as proxy_script:
+ proxy_script.write("proxy script\n")
+ os.fchmod(proxy_script.fileno(), 0o755)
+ build_source.install()
+ self.assertThat(
+ build_source.backend.run.calls,
+ MatchesAll(
+ AnyMatch(
+ RanCommand(
+ ["git", "config", "--global", "protocol.version", "2"]
+ )
+ ),
+ ),
+ )
+
def test_repo_bzr(self):
args = [
"build-source",
@@ -523,6 +794,79 @@ class TestBuildSource(TestCase):
with open(status_path) as status:
self.assertEqual({"revision_id": "0" * 40}, json.load(status))
+ def test_repo_fetch_service(self):
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--git-repository",
+ "lp:foo",
+ "--proxy-url",
+ "http://proxy.example:3128/";,
+ "test-image",
+ "--use_fetch_service",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.backend.build_path = self.useFixture(TempDir()).path
+ build_source.backend.run = FakeRevisionID("0" * 40)
+ build_source.repo()
+ env = {
+ "http_proxy": "http://proxy.example:3128/";,
+ "https_proxy": "http://proxy.example:3128/";,
+ "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy",
+ "SNAPPY_STORE_NO_CDN": "1",
+ }
+ self.assertThat(
+ build_source.backend.run.calls,
+ MatchesListwise(
+ [
+ RanBuildCommand(
+ [
+ "git",
+ "clone",
+ "-n",
+ "--depth",
+ "1",
+ "-b",
+ "HEAD",
+ "--single-branch",
+ "lp:foo",
+ "test-image",
+ ],
+ cwd="/home/buildd",
+ **env,
+ ),
+ RanBuildCommand(
+ ["git", "checkout", "-q", "HEAD"],
+ cwd="/home/buildd/test-image",
+ **env,
+ ),
+ RanBuildCommand(
+ [
+ "git",
+ "submodule",
+ "update",
+ "--init",
+ "--recursive",
+ ],
+ cwd="/home/buildd/test-image",
+ **env,
+ ),
+ RanBuildCommand(
+ ["git", "rev-parse", "HEAD^{}"],
+ cwd="/home/buildd/test-image",
+ get_output=True,
+ universal_newlines=True,
+ ),
+ ]
+ ),
+ )
+ status_path = os.path.join(build_source.backend.build_path, "status")
+ with open(status_path) as status:
+ self.assertEqual({"revision_id": "0" * 40}, json.load(status))
+
def test_build(self):
args = [
"build-source",
@@ -611,6 +955,44 @@ class TestBuildSource(TestCase):
),
)
+ def test_build_fetch_service(self):
+ args = [
+ "build-source",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--branch",
+ "lp:foo",
+ "--proxy-url",
+ "http://proxy.example:3128/";,
+ "test-image",
+ "--use_fetch_service",
+ "--fetch-service-mitm-certificate",
+ # Base64 content_of_cert
+ "Y29udGVudF9vZl9jZXJ0",
+ ]
+ build_source = parse_args(args=args).operation
+ build_source.build()
+ env = {
+ "http_proxy": "http://proxy.example:3128/";,
+ "https_proxy": "http://proxy.example:3128/";,
+ "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy",
+ "SNAPPY_STORE_NO_CDN": "1",
+ }
+ self.assertThat(
+ build_source.backend.run.calls,
+ MatchesListwise(
+ [
+ RanBuildCommand(
+ ["sourcecraft", "pack", "-v", "--destructive-mode"],
+ cwd="/home/buildd/test-image/.",
+ **env,
+ ),
+ ]
+ ),
+ )
+
def test_run_succeeds(self):
args = [
"build-source",
diff --git a/lpbuildd/tests/test_source.py b/lpbuildd/tests/test_source.py
index 6382be5..8fa3c87 100644
--- a/lpbuildd/tests/test_source.py
+++ b/lpbuildd/tests/test_source.py
@@ -239,3 +239,18 @@ class TestSourceBuildManagerIteration(TestCase):
self.buildmanager.iterate, self.buildmanager.iterators[-1]
)
self.assertFalse(self.builder.wasCalled("buildFail"))
+
+ @defer.inlineCallbacks
+ def test_iterate_use_fetch_service(self):
+ # The build manager can be told to use the fetch service as its proxy.
+ # This requires also a ca certificate passed in via secrets.
+ args = {
+ "use_fetch_service": True,
+ "secrets": {"fetch_service_mitm_certificate": "content_of_cert"},
+ }
+ expected_options = [
+ "--use_fetch_service",
+ "--fetch-service-mitm-certificate",
+ "content_of_cert",
+ ]
+ yield self.startBuild(args, expected_options)
