launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #32528
[Merge] ~alvarocs/launchpad-buildd:fetch-service-charm-builds into launchpad-buildd:master
Alvaro Crespo Serrano has proposed merging ~alvarocs/launchpad-buildd:fetch-service-charm-builds into launchpad-buildd:master.
Commit message:
Enable fetch service support for charm builds
This adds support for the --use-fetch-service flag in CharmBuildManager,
allowing charm builds to use the fetch service and MITM certificate when
requested. This aligns charm build behavior with existing support for
rocks and snaps.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~alvarocs/launchpad-buildd/+git/launchpad-buildd/+merge/486058
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~alvarocs/launchpad-buildd:fetch-service-charm-builds into launchpad-buildd:master.
diff --git a/debian/changelog b/debian/changelog
index 8b14d6d..f72cb19 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+launchpad-buildd (256) noble; urgency=medium
+
+ [ Alvaro Crespo ]
+ * Enable fetch service for charm builds.
+
+ -- Alvaro Crespo <alvaro.crespo@xxxxxxxxxxxxx> Mon, 19 May 2024 17:47:21 +0200
+
launchpad-buildd (255) noble; urgency=medium
[ Alvaro Crespo ]
diff --git a/lpbuildd/charm.py b/lpbuildd/charm.py
index e79aacb..6101d96 100644
--- a/lpbuildd/charm.py
+++ b/lpbuildd/charm.py
@@ -37,6 +37,8 @@ class CharmBuildManager(BuildManagerProxyMixin, DebianBuildManager):
self.revocation_endpoint = extra_args.get("revocation_endpoint")
self.proxy_service = None
self.craft_platform = extra_args.get("craft_platform")
+ self.secrets = extra_args.get("secrets")
+ self.use_fetch_service= extra_args.get("use_fetch_service")
super().initiate(files, chroot, extra_args)
@@ -58,6 +60,14 @@ class CharmBuildManager(BuildManagerProxyMixin, DebianBuildManager):
args.extend(["--build-path", self.build_path])
if self.craft_platform:
args.extend(["--craft-platform", self.craft_platform])
+ if self.use_fetch_service:
+ args.append("--use-fetch-service")
+ args.extend(
+ [
+ "--fetch-service-mitm-certificate",
+ self.secrets["fetch_service_mitm_certificate"],
+ ]
+ )
args.append(self.name)
self.runTargetSubProcess("build-charm", *args)
diff --git a/lpbuildd/target/build_charm.py b/lpbuildd/target/build_charm.py
index ec82b9d..95dade1 100644
--- a/lpbuildd/target/build_charm.py
+++ b/lpbuildd/target/build_charm.py
@@ -46,6 +46,17 @@ class BuildCharm(
type=str,
help="craft platform name used by the craft tool"
)
+ parser.add_argument(
+ "--use-fetch-service",
+ default=False,
+ action="store_true",
+ help="use the fetch service instead of the builder proxy",
+ )
+ parser.add_argument(
+ "--fetch-service-mitm-certificate",
+ type=str,
+ help="content of the ca certificate",
+ )
def __init__(self, args, parser):
super().__init__(args, parser)
@@ -92,6 +103,17 @@ class BuildCharm(
)
else:
self.backend.run(["snap", "install", "--classic", "charmcraft"])
+
+ if self.args.use_fetch_service:
+ # Deleting apt cache /var/lib/apt/lists before
+ # installing the fetch service
+ self.install_apt_proxy()
+ self.delete_apt_cache()
+ self.install_mitm_certificate()
+ self.install_snapd_proxy(proxy_url=self.args.proxy_url)
+ self.backend.run(["apt-get", "-y", "update"])
+ self.restart_snapd()
+ self.configure_git_protocol_v2()
# The charmcraft snap can't see /build, so we have to do our work under
# /home/buildd instead. Make sure it exists.
self.backend.run(["mkdir", "-p", "/home/buildd"])
@@ -99,8 +121,18 @@ class BuildCharm(
def repo(self):
"""Collect git or bzr branch."""
logger.info("Running repo phase...")
- env = self.build_proxy_environment(proxy_url=self.args.proxy_url)
- self.vcs_fetch(self.args.name, cwd="/home/buildd", env=env)
+ env = self.build_proxy_environment(
+ proxy_url=self.args.proxy_url,
+ use_fetch_service=self.args.use_fetch_service
+ )
+ # using the fetch service requires shallow clones
+ git_shallow_clone = bool(self.args.use_fetch_service)
+ self.vcs_fetch(
+ self.args.name,
+ cwd="/home/buildd",
+ env=env,
+ git_shallow_clone_with_single_branch=git_shallow_clone,
+ )
self.vcs_update_status(self.buildd_path)
def build(self):
@@ -109,7 +141,10 @@ class BuildCharm(
"/home/buildd", self.args.name, self.args.build_path
)
check_path_escape(self.buildd_path, build_context_path)
- env = self.build_proxy_environment(proxy_url=self.args.proxy_url)
+ env = self.build_proxy_environment(
+ proxy_url=self.args.proxy_url,
+ use_fetch_service=self.args.use_fetch_service,
+ )
if self.args.craft_platform:
env["CRAFT_PLATFORM"] = self.args.craft_platform
args = ["charmcraft", "pack", "-v", "--destructive-mode"]
diff --git a/lpbuildd/target/tests/test_build_charm.py b/lpbuildd/target/tests/test_build_charm.py
index 6c3c04a..42e3154 100644
--- a/lpbuildd/target/tests/test_build_charm.py
+++ b/lpbuildd/target/tests/test_build_charm.py
@@ -10,7 +10,7 @@ from textwrap import dedent
import responses
from fixtures import FakeLogger, TempDir
from systemfixtures import FakeFilesystem
-from testtools.matchers import AnyMatch, MatchesAll, MatchesListwise
+from testtools.matchers import AnyMatch, MatchesAll, MatchesListwise, Not
from testtools.testcase import TestCase
from lpbuildd.target.backend import InvalidBuildFilePath
@@ -265,6 +265,267 @@ class TestBuildCharm(TestCase):
],
)
+ def test_install_certificate(self):
+ args = [
+ "build-charm",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--git-repository",
+ "lp:foo",
+ "--proxy-url",
+ "http://proxy.example:3128/";,
+ "test-image",
+ "--use-fetch-service",
+ "--fetch-service-mitm-certificate",
+ # Base64 content_of_cert
+ "Y29udGVudF9vZl9jZXJ0",
+ ]
+ build_charm = parse_args(args=args).operation
+ build_charm.bin = "/builderbin"
+ self.useFixture(FakeFilesystem()).add("/builderbin")
+ os.mkdir("/builderbin")
+ with open("/builderbin/lpbuildd-git-proxy", "w") as proxy_script:
+ proxy_script.write("proxy script\n")
+ os.fchmod(proxy_script.fileno(), 0o755)
+ build_charm.install()
+ self.assertThat(
+ build_charm.backend.run.calls,
+ MatchesListwise(
+ [
+ RanAptGet(
+ "install",
+ "python3",
+ "socat",
+ "git",
+ "python3-pip",
+ "python3-setuptools",
+ ),
+ RanSnap("install", "--classic", "charmcraft"),
+ RanCommand(["rm", "-rf", "/var/lib/apt/lists"]),
+ RanCommand(["update-ca-certificates"]),
+ RanCommand(
+ [
+ "snap",
+ "set",
+ "system",
+ "proxy.http=http://proxy.example:3128/";,
+ ]
+ ),
+ RanCommand(
+ [
+ "snap",
+ "set",
+ "system",
+ "proxy.https=http://proxy.example:3128/";,
+ ]
+ ),
+ RanAptGet("update"),
+ RanCommand(
+ [
+ "systemctl",
+ "restart",
+ "snapd",
+ ]
+ ),
+ RanCommand(["mkdir", "-p", "/home/buildd"]),
+ ]
+ ),
+ )
+ self.assertEqual(
+ (b"proxy script\n", stat.S_IFREG | 0o755),
+ build_charm.backend.backend_fs["/usr/local/bin/lpbuildd-git-proxy"],
+ )
+ self.assertEqual(
+ (
+ b"content_of_cert",
+ stat.S_IFREG | 0o644,
+ ),
+ build_charm.backend.backend_fs[
+ "/usr/local/share/ca-certificates/local-ca.crt"
+ ],
+ )
+ self.assertEqual(
+ (
+ dedent(
+ """\
+ Acquire::http::Proxy "http://proxy.example:3128/";;
+ Acquire::https::Proxy "http://proxy.example:3128/";;
+
+ """
+ ).encode("UTF-8"),
+ stat.S_IFREG | 0o644,
+ ),
+ build_charm.backend.backend_fs["/etc/apt/apt.conf.d/99proxy"],
+ )
+
+ def test_install_snapd_proxy(self):
+ args = [
+ "build-charm",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--git-repository",
+ "lp:foo",
+ "--proxy-url",
+ "http://proxy.example:3128/";,
+ "test-image",
+ "--use-fetch-service",
+ "--fetch-service-mitm-certificate",
+ # Base64 content_of_cert
+ "Y29udGVudF9vZl9jZXJ0",
+ ]
+ build_charm = parse_args(args=args).operation
+ build_charm.bin = "/builderbin"
+ self.useFixture(FakeFilesystem()).add("/builderbin")
+ os.mkdir("/builderbin")
+ with open("/builderbin/lpbuildd-git-proxy", "w") as proxy_script:
+ proxy_script.write("proxy script\n")
+ os.fchmod(proxy_script.fileno(), 0o755)
+ build_charm.install()
+ self.assertThat(
+ build_charm.backend.run.calls,
+ MatchesListwise(
+ [
+ RanAptGet(
+ "install",
+ "python3",
+ "socat",
+ "git",
+ "python3-pip",
+ "python3-setuptools",
+ ),
+ RanSnap("install", "--classic", "charmcraft"),
+ RanCommand(["rm", "-rf", "/var/lib/apt/lists"]),
+ RanCommand(["update-ca-certificates"]),
+ RanCommand(
+ [
+ "snap",
+ "set",
+ "system",
+ "proxy.http=http://proxy.example:3128/";,
+ ]
+ ),
+ RanCommand(
+ [
+ "snap",
+ "set",
+ "system",
+ "proxy.https=http://proxy.example:3128/";,
+ ]
+ ),
+ RanAptGet("update"),
+ RanCommand(
+ [
+ "systemctl",
+ "restart",
+ "snapd",
+ ]
+ ),
+ RanCommand(["mkdir", "-p", "/home/buildd"]),
+ ]
+ ),
+ )
+ self.assertEqual(
+ (b"proxy script\n", stat.S_IFREG | 0o755),
+ build_charm.backend.backend_fs["/usr/local/bin/lpbuildd-git-proxy"],
+ )
+ self.assertEqual(
+ (
+ dedent(
+ """\
+ Acquire::http::Proxy "http://proxy.example:3128/";;
+ Acquire::https::Proxy "http://proxy.example:3128/";;
+
+ """
+ ).encode("UTF-8"),
+ stat.S_IFREG | 0o644,
+ ),
+ build_charm.backend.backend_fs["/etc/apt/apt.conf.d/99proxy"],
+ )
+
+ def test_install_fetch_service(self):
+ args = [
+ "build-charm",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--git-repository",
+ "lp:foo",
+ "--proxy-url",
+ "http://proxy.example:3128/";,
+ "test-image",
+ "--use-fetch-service",
+ "--fetch-service-mitm-certificate",
+ # Base64 content_of_cert
+ "Y29udGVudF9vZl9jZXJ0",
+ ]
+ build_charm = parse_args(args=args).operation
+ build_charm.bin = "/builderbin"
+ self.useFixture(FakeFilesystem()).add("/builderbin")
+ os.mkdir("/builderbin")
+ with open("/builderbin/lpbuildd-git-proxy", "w") as proxy_script:
+ proxy_script.write("proxy script\n")
+ os.fchmod(proxy_script.fileno(), 0o755)
+ build_charm.install()
+ self.assertThat(
+ build_charm.backend.run.calls,
+ MatchesAll(
+ Not(
+ AnyMatch(
+ RanCommand(
+ [
+ "git",
+ "config",
+ "--global",
+ "protocol.version",
+ "2",
+ ]
+ )
+ )
+ ),
+ ),
+ )
+
+ def test_install_fetch_service_focal(self):
+ args = [
+ "build-charm",
+ "--backend=fake",
+ "--series=focal",
+ "--arch=amd64",
+ "1",
+ "--git-repository",
+ "lp:foo",
+ "--proxy-url",
+ "http://proxy.example:3128/";,
+ "test-image",
+ "--use-fetch-service",
+ "--fetch-service-mitm-certificate",
+ # Base64 content_of_cert
+ "Y29udGVudF9vZl9jZXJ0",
+ ]
+ build_charm = parse_args(args=args).operation
+ build_charm.bin = "/builderbin"
+ self.useFixture(FakeFilesystem()).add("/builderbin")
+ os.mkdir("/builderbin")
+ with open("/builderbin/lpbuildd-git-proxy", "w") as proxy_script:
+ proxy_script.write("proxy script\n")
+ os.fchmod(proxy_script.fileno(), 0o755)
+ build_charm.install()
+ self.assertThat(
+ build_charm.backend.run.calls,
+ MatchesAll(
+ AnyMatch(
+ RanCommand(
+ ["git", "config", "--global", "protocol.version", "2"]
+ )
+ ),
+ ),
+ )
+
def test_repo_bzr(self):
args = [
"build-charm",
@@ -518,6 +779,88 @@ class TestBuildCharm(TestCase):
with open(status_path) as status:
self.assertEqual({"revision_id": "0" * 40}, json.load(status))
+ def test_repo_fetch_service(self):
+ args = [
+ "build-charm",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--git-repository",
+ "lp:foo",
+ "--proxy-url",
+ "http://proxy.example:3128/";,
+ "test-image",
+ "--use-fetch-service",
+ ]
+ build_charm = parse_args(args=args).operation
+ build_charm.backend.build_path = self.useFixture(TempDir()).path
+ build_charm.backend.run = FakeRevisionID("0" * 40)
+ build_charm.repo()
+ env = {
+ "http_proxy": "http://proxy.example:3128/";,
+ "HTTP_PROXY": "http://proxy.example:3128/";,
+ "https_proxy": "http://proxy.example:3128/";,
+ "HTTPS_PROXY": "http://proxy.example:3128/";,
+ "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy",
+ "SNAPPY_STORE_NO_CDN": "1",
+ "CARGO_HTTP_CAINFO": (
+ "/usr/local/share/ca-certificates/local-ca.crt"
+ ),
+ "REQUESTS_CA_BUNDLE": (
+ "/usr/local/share/ca-certificates/local-ca.crt"
+ ),
+ "GOPROXY": "direct",
+ }
+ self.assertThat(
+ build_charm.backend.run.calls,
+ MatchesListwise(
+ [
+ RanBuildCommand(
+ [
+ "git",
+ "clone",
+ "-n",
+ "--depth",
+ "1",
+ "-b",
+ "HEAD",
+ "--single-branch",
+ "lp:foo",
+ "test-image",
+ ],
+ cwd="/home/buildd",
+ **env,
+ ),
+ RanBuildCommand(
+ ["git", "checkout", "-q", "HEAD"],
+ cwd="/home/buildd/test-image",
+ **env,
+ ),
+ RanBuildCommand(
+ [
+ "git",
+ "submodule",
+ "update",
+ "--init",
+ "--recursive",
+ ],
+ cwd="/home/buildd/test-image",
+ **env,
+ ),
+ RanBuildCommand(
+ ["git", "rev-parse", "HEAD^{}"],
+ cwd="/home/buildd/test-image",
+ get_output=True,
+ universal_newlines=True,
+ ),
+ ]
+ ),
+ )
+ status_path = os.path.join(build_charm.backend.build_path, "status")
+ with open(status_path) as status:
+ self.assertEqual({"revision_id": "0" * 40}, json.load(status))
+
def test_build(self):
args = [
"build-charm",
@@ -636,6 +979,53 @@ class TestBuildCharm(TestCase):
]
),
)
+
+ def test_build_fetch_service(self):
+ args = [
+ "build-charm",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--branch",
+ "lp:foo",
+ "--proxy-url",
+ "http://proxy.example:3128/";,
+ "test-image",
+ "--use-fetch-service",
+ "--fetch-service-mitm-certificate",
+ # Base64 content_of_cert
+ "Y29udGVudF9vZl9jZXJ0",
+ ]
+ build_charm = parse_args(args=args).operation
+ build_charm.build()
+ env = {
+ "http_proxy": "http://proxy.example:3128/";,
+ "HTTP_PROXY": "http://proxy.example:3128/";,
+ "https_proxy": "http://proxy.example:3128/";,
+ "HTTPS_PROXY": "http://proxy.example:3128/";,
+ "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy",
+ "SNAPPY_STORE_NO_CDN": "1",
+ "CARGO_HTTP_CAINFO": (
+ "/usr/local/share/ca-certificates/local-ca.crt"
+ ),
+ "REQUESTS_CA_BUNDLE": (
+ "/usr/local/share/ca-certificates/local-ca.crt"
+ ),
+ "GOPROXY": "direct",
+ }
+ self.assertThat(
+ build_charm.backend.run.calls,
+ MatchesListwise(
+ [
+ RanBuildCommand(
+ ["charmcraft", "pack", "-v", "--destructive-mode"],
+ cwd="/home/buildd/test-image/.",
+ **env,
+ ),
+ ]
+ ),
+ )
def test_run_succeeds(self):
args = [
diff --git a/lpbuildd/tests/test_charm.py b/lpbuildd/tests/test_charm.py
index b8a5183..3944ab4 100644
--- a/lpbuildd/tests/test_charm.py
+++ b/lpbuildd/tests/test_charm.py
@@ -243,6 +243,21 @@ class TestCharmBuildManagerIteration(TestCase):
self.buildmanager.iterate, self.buildmanager.iterators[-1]
)
self.assertFalse(self.builder.wasCalled("buildFail"))
+
+ @defer.inlineCallbacks
+ def test_iterate_use_fetch_service(self):
+ # The build manager can be told to use the fetch service as its proxy.
+ # This requires also a ca certificate passed in via secrets.
+ args = {
+ "use_fetch_service": True,
+ "secrets": {"fetch_service_mitm_certificate": "content_of_cert"},
+ }
+ expected_options = [
+ "--use-fetch-service",
+ "--fetch-service-mitm-certificate",
+ "content_of_cert",
+ ]
+ yield self.startBuild(args, expected_options)
@defer.inlineCallbacks
def test_iterate_craft_platform(self):
Follow ups
