launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #27263
[Merge] ~cjwatson/launchpad-buildd:charm-snap-build-proxy into launchpad-buildd:master
Colin Watson has proposed merging ~cjwatson/launchpad-buildd:charm-snap-build-proxy into launchpad-buildd:master.
Commit message:
Honour proxy arguments when building charms
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad-buildd/+git/launchpad-buildd/+merge/405623
charmcraft essentially always runs pip3 as part of building the charm, and has no obvious way to provide a local package index. It looks as though we'll need to regard charms as requiring internet access to build.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad-buildd:charm-snap-build-proxy into launchpad-buildd:master.
diff --git a/debian/changelog b/debian/changelog
index e66175d..0dcf32e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
launchpad-buildd (198) UNRELEASED; urgency=medium
* Run charmcraft in verbose mode.
+ * Honour proxy arguments when building charms.
-- Colin Watson <cjwatson@xxxxxxxxxx> Fri, 09 Jul 2021 14:08:58 +0100
diff --git a/lpbuildd/charm.py b/lpbuildd/charm.py
index e5c5fe9..0838fe7 100644
--- a/lpbuildd/charm.py
+++ b/lpbuildd/charm.py
@@ -11,6 +11,7 @@ from lpbuildd.debian import (
DebianBuildState,
DebianBuildManager,
)
+from lpbuildd.snap import SnapBuildProxyMixin
RETCODE_SUCCESS = 0
@@ -22,7 +23,7 @@ class CharmBuildState(DebianBuildState):
BUILD_CHARM = "BUILD_CHARM"
-class CharmBuildManager(DebianBuildManager):
+class CharmBuildManager(SnapBuildProxyMixin, DebianBuildManager):
"""Build a charm."""
backend_name = "lxd"
@@ -40,6 +41,9 @@ class CharmBuildManager(DebianBuildManager):
self.git_path = extra_args.get("git_path")
self.build_path = extra_args.get("build_path")
self.channels = extra_args.get("channels", {})
+ self.proxy_url = extra_args.get("proxy_url")
+ self.revocation_endpoint = extra_args.get("revocation_endpoint")
+ self.proxy_service = None
super(CharmBuildManager, self).initiate(files, chroot, extra_args)
diff --git a/lpbuildd/target/build_charm.py b/lpbuildd/target/build_charm.py
index 95da85e..7610a8b 100644
--- a/lpbuildd/target/build_charm.py
+++ b/lpbuildd/target/build_charm.py
@@ -2,7 +2,6 @@
# GNU Affero General Public License version 3 (see the file LICENSE).
from __future__ import print_function
-import functools
__metaclass__ = type
@@ -14,6 +13,7 @@ import sys
from lpbuildd.target.backend import check_path_escape
from lpbuildd.target.build_snap import SnapChannelsAction
from lpbuildd.target.operation import Operation
+from lpbuildd.target.snapbuildproxy import SnapBuildProxyOperationMixin
from lpbuildd.target.snapstore import SnapStoreOperationMixin
from lpbuildd.target.vcs import VCSOperationMixin
@@ -25,7 +25,8 @@ RETCODE_FAILURE_BUILD = 201
logger = logging.getLogger(__name__)
-class BuildCharm(VCSOperationMixin, SnapStoreOperationMixin, Operation):
+class BuildCharm(SnapBuildProxyOperationMixin, VCSOperationMixin,
+ SnapStoreOperationMixin, Operation):
description = "Build a charm."
@@ -69,6 +70,9 @@ class BuildCharm(VCSOperationMixin, SnapStoreOperationMixin, Operation):
def install(self):
logger.info("Running install phase")
deps = []
+ if self.args.proxy_url:
+ deps.extend(self.proxy_deps)
+ self.install_git_proxy()
if self.args.backend == "lxd":
# udev is installed explicitly to work around
# https://bugs.launchpad.net/snapd/+bug/1731519.
@@ -99,7 +103,8 @@ class BuildCharm(VCSOperationMixin, SnapStoreOperationMixin, Operation):
def repo(self):
"""Collect git or bzr branch."""
logger.info("Running repo phase...")
- self.vcs_fetch(self.args.name, cwd="/home/buildd")
+ env = self.build_proxy_environment(proxy_url=self.args.proxy_url)
+ self.vcs_fetch(self.args.name, cwd="/home/buildd", env=env)
self.save_status(self.buildd_path)
def build(self):
@@ -109,8 +114,13 @@ class BuildCharm(VCSOperationMixin, SnapStoreOperationMixin, Operation):
self.args.name,
self.args.build_path)
check_path_escape(self.buildd_path, build_context_path)
+ env = OrderedDict()
+ if self.args.proxy_url:
+ env["http_proxy"] = self.args.proxy_url
+ env["https_proxy"] = self.args.proxy_url
+ env["GIT_PROXY_COMMAND"] = "/usr/local/bin/snap-git-proxy"
args = ["charmcraft", "build", "-v", "-f", build_context_path]
- self.run_build_command(args)
+ self.run_build_command(args, env=env)
def run(self):
try:
diff --git a/lpbuildd/target/tests/test_build_charm.py b/lpbuildd/target/tests/test_build_charm.py
index 4e77943..8be840e 100644
--- a/lpbuildd/target/tests/test_build_charm.py
+++ b/lpbuildd/target/tests/test_build_charm.py
@@ -5,6 +5,7 @@ __metaclass__ = type
import json
import os
+import stat
import subprocess
from textwrap import dedent
@@ -13,6 +14,7 @@ from fixtures import (
TempDir,
)
import responses
+from systemfixtures import FakeFilesystem
from testtools.matchers import (
AnyMatch,
Equals,
@@ -171,11 +173,11 @@ class TestBuildCharm(TestCase):
"--backend=fake", "--series=xenial", "--arch=amd64", "1",
"--git-repository", "lp:foo",
"--snap-store-proxy-url", "http://snap-store-proxy.example/";,
- "test-snap",
+ "test-image",
]
- build_snap = parse_args(args=args).operation
- build_snap.install()
- self.assertThat(build_snap.backend.run.calls, MatchesListwise([
+ build_charm = parse_args(args=args).operation
+ build_charm.install()
+ self.assertThat(build_charm.backend.run.calls, MatchesListwise([
RanAptGet("install", "git"),
RanCommand(
["snap", "ack", "/dev/stdin"], input_text=store_assertion),
@@ -184,6 +186,31 @@ class TestBuildCharm(TestCase):
RanCommand(["mkdir", "-p", "/home/buildd"]),
]))
+ def test_install_proxy(self):
+ args = [
+ "build-charm",
+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+ "--git-repository", "lp:foo",
+ "--proxy-url", "http://proxy.example:3128/";,
+ "test-image",
+ ]
+ build_charm = parse_args(args=args).operation
+ build_charm.bin = "/builderbin"
+ self.useFixture(FakeFilesystem()).add("/builderbin")
+ os.mkdir("/builderbin")
+ with open("/builderbin/snap-git-proxy", "w") as proxy_script:
+ proxy_script.write("proxy script\n")
+ os.fchmod(proxy_script.fileno(), 0o755)
+ build_charm.install()
+ self.assertThat(build_charm.backend.run.calls, MatchesListwise([
+ RanAptGet("install", "python3", "socat", "git"),
+ RanSnap("install", "charmcraft"),
+ RanCommand(["mkdir", "-p", "/home/buildd"]),
+ ]))
+ self.assertEqual(
+ (b"proxy script\n", stat.S_IFREG | 0o755),
+ build_charm.backend.backend_fs["/usr/local/bin/snap-git-proxy"])
+
def test_repo_bzr(self):
args = [
"build-charm",
@@ -284,6 +311,39 @@ class TestBuildCharm(TestCase):
with open(status_path) as status:
self.assertEqual({"revision_id": "0" * 40}, json.load(status))
+ def test_repo_proxy(self):
+ args = [
+ "build-charm",
+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+ "--git-repository", "lp:foo",
+ "--proxy-url", "http://proxy.example:3128/";,
+ "test-image",
+ ]
+ build_charm = parse_args(args=args).operation
+ build_charm.backend.build_path = self.useFixture(TempDir()).path
+ build_charm.backend.run = FakeRevisionID("0" * 40)
+ build_charm.repo()
+ env = {
+ "http_proxy": "http://proxy.example:3128/";,
+ "https_proxy": "http://proxy.example:3128/";,
+ "GIT_PROXY_COMMAND": "/usr/local/bin/snap-git-proxy",
+ }
+ self.assertThat(build_charm.backend.run.calls, MatchesListwise([
+ RanBuildCommand(
+ ["git", "clone", "lp:foo", "test-image"],
+ cwd="/home/buildd", **env),
+ RanBuildCommand(
+ ["git", "submodule", "update", "--init", "--recursive"],
+ cwd="/home/buildd/test-image", **env),
+ RanBuildCommand(
+ ["git", "rev-parse", "HEAD^{}"],
+ cwd="/home/buildd/test-image", get_output=True,
+ universal_newlines=True),
+ ]))
+ status_path = os.path.join(build_charm.backend.build_path, "status")
+ with open(status_path) as status:
+ self.assertEqual({"revision_id": "0" * 40}, json.load(status))
+
def test_build(self):
args = [
"build-charm",
@@ -317,6 +377,27 @@ class TestBuildCharm(TestCase):
cwd="/home/buildd/test-image"),
]))
+ def test_build_proxy(self):
+ args = [
+ "build-charm",
+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+ "--branch", "lp:foo", "--proxy-url", "http://proxy.example:3128/";,
+ "test-image",
+ ]
+ build_charm = parse_args(args=args).operation
+ build_charm.build()
+ env = {
+ "http_proxy": "http://proxy.example:3128/";,
+ "https_proxy": "http://proxy.example:3128/";,
+ "GIT_PROXY_COMMAND": "/usr/local/bin/snap-git-proxy",
+ }
+ self.assertThat(build_charm.backend.run.calls, MatchesListwise([
+ RanBuildCommand(
+ ["charmcraft", "build", "-v", "-f",
+ "/home/buildd/test-image/."],
+ cwd="/home/buildd/test-image", **env),
+ ]))
+
def test_run_succeeds(self):
args = [
"build-charm",
diff --git a/lpbuildd/tests/test_charm.py b/lpbuildd/tests/test_charm.py
index 407f732..8610bad 100644
--- a/lpbuildd/tests/test_charm.py
+++ b/lpbuildd/tests/test_charm.py
@@ -4,6 +4,10 @@
__metaclass__ = type
import os
+try:
+ from unittest import mock
+except ImportError:
+ import mock
from fixtures import (
EnvironmentVariable,
@@ -137,3 +141,15 @@ class TestCharmBuildManagerIteration(TestCase):
self.assertEqual(
self.buildmanager.iterate, self.buildmanager.iterators[-1])
self.assertFalse(self.builder.wasCalled("buildFail"))
+
+ @mock.patch('lpbuildd.snap.urlopen')
+ def test_revokeProxyToken(self, urlopen_mock):
+ self.buildmanager.revocation_endpoint = "http://revoke_endpoint";
+ self.buildmanager.proxy_url = "http://username:password@proxy_url";
+ self.buildmanager.revokeProxyToken()
+ self.assertEqual(1, urlopen_mock.call_count)
+ request = urlopen_mock.call_args[0][0]
+ self.assertEqual(
+ {'Authorization': "Basic dXNlcm5hbWU6cGFzc3dvcmQ="},
+ request.headers)
+ self.assertEqual('http://revoke_endpoint', request.get_full_url())
