launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #22244
[Merge] lp:~cjwatson/launchpad-buildd/refactor-vcs into lp:launchpad-buildd
Colin Watson has proposed merging lp:~cjwatson/launchpad-buildd/refactor-vcs into lp:launchpad-buildd.
Commit message:
Refactor VCS operations from lpbuildd.target.build_snap out to a module that can be used by other targets.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad-buildd/refactor-vcs/+merge/341388
Multiple targets have a VCS source of some kind, and there's no particular reason for them to do it differently.
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad-buildd/refactor-vcs into lp:launchpad-buildd.
=== modified file 'debian/changelog'
--- debian/changelog 2018-02-04 00:16:23 +0000
+++ debian/changelog 2018-03-14 14:06:41 +0000
@@ -5,6 +5,8 @@
won't start any of them, including snapd itself, if any of them is
forbidden. Mask snapd.refresh.timer so that it doesn't cause trouble.
* Allow optionally installing snapcraft as a snap (LP: #1737994).
+ * Refactor VCS operations from lpbuildd.target.build_snap out to a module
+ that can be used by other targets.
-- Colin Watson <cjwatson@xxxxxxxxxx> Fri, 02 Feb 2018 16:14:46 +0000
=== modified file 'lpbuildd/target/build_snap.py'
--- lpbuildd/target/build_snap.py 2018-02-04 00:16:23 +0000
+++ lpbuildd/target/build_snap.py 2018-03-14 14:06:41 +0000
@@ -10,7 +10,6 @@
import json
import logging
import os.path
-import subprocess
import sys
try:
from urllib.error import (
@@ -32,6 +31,7 @@
from urlparse import urlparse
from lpbuildd.target.operation import Operation
+from lpbuildd.target.vcs import VCSOperationMixin
RETCODE_FAILURE_INSTALL = 200
@@ -41,7 +41,7 @@
logger = logging.getLogger(__name__)
-class BuildSnap(Operation):
+class BuildSnap(VCSOperationMixin, Operation):
description = "Build a snap."
@@ -56,15 +56,6 @@
help=(
"install snapcraft as a snap from CHANNEL rather than as a "
".deb"))
- build_from_group = parser.add_mutually_exclusive_group(required=True)
- build_from_group.add_argument(
- "--branch", metavar="BRANCH", help="build from this Bazaar branch")
- build_from_group.add_argument(
- "--git-repository", metavar="REPOSITORY",
- help="build from this Git repository")
- parser.add_argument(
- "--git-path", metavar="REF-PATH",
- help="build from this ref path in REPOSITORY")
parser.add_argument("--proxy-url", help="builder proxy url")
parser.add_argument(
"--revocation-endpoint",
@@ -73,18 +64,12 @@
def __init__(self, args, parser):
super(BuildSnap, self).__init__(args, parser)
- if args.git_repository is None and args.git_path is not None:
- parser.error("--git-path requires --git-repository")
self.slavebin = os.path.dirname(sys.argv[0])
- # Set to False for local testing if your target doesn't have an
- # appropriate certificate for your codehosting system.
- self.ssl_verify = True
- def run_build_command(self, args, cwd="/build", env=None, **kwargs):
+ def run_build_command(self, args, env=None, **kwargs):
"""Run a build command in the target.
:param args: the command and arguments to run.
- :param cwd: run the command in this working directory in the target.
:param env: dictionary of additional environment variables to set.
:param kwargs: any other keyword arguments to pass to Backend.run.
"""
@@ -93,7 +78,7 @@
full_env["SHELL"] = "/bin/sh"
if env:
full_env.update(env)
- return self.backend.run(args, cwd=cwd, env=full_env, **kwargs)
+ return self.backend.run(args, env=full_env, **kwargs)
def save_status(self, status):
"""Save a dictionary of status information about this build.
@@ -115,10 +100,7 @@
for dep in "snapd", "fuse", "squashfuse", "udev":
if self.backend.is_package_available(dep):
deps.append(dep)
- if self.args.branch is not None:
- deps.append("bzr")
- else:
- deps.append("git")
+ deps.extend(self.vcs_deps)
if self.args.proxy_url:
deps.extend(["python3", "socat"])
if not self.args.channel_snapcraft:
@@ -145,41 +127,20 @@
env["http_proxy"] = self.args.proxy_url
env["https_proxy"] = self.args.proxy_url
env["GIT_PROXY_COMMAND"] = "/usr/local/bin/snap-git-proxy"
- if self.args.branch is not None:
- self.run_build_command(['ls', '/build'])
- cmd = ["bzr", "branch", self.args.branch, self.args.name]
- if not self.ssl_verify:
- cmd.insert(1, "-Ossl.cert_reqs=none")
- else:
- assert self.args.git_repository is not None
- cmd = ["git", "clone"]
- if self.args.git_path is not None:
- cmd.extend(["-b", self.args.git_path])
- cmd.extend([self.args.git_repository, self.args.name])
- if not self.ssl_verify:
- env["GIT_SSL_NO_VERIFY"] = "1"
- self.run_build_command(cmd, env=env)
- if self.args.git_repository is not None:
- try:
- self.run_build_command(
- ["git", "-C", self.args.name,
- "submodule", "update", "--init", "--recursive"],
- env=env)
- except subprocess.CalledProcessError as e:
- logger.error(
- "'git submodule update --init --recursive failed with "
- "exit code %s (build may fail later)" % e.returncode)
+ self.vcs_fetch(self.args.name, cwd="/build", env=env)
status = {}
if self.args.branch is not None:
status["revision_id"] = self.run_build_command(
- ["bzr", "revno", self.args.name],
+ ["bzr", "revno"],
+ cwd=os.path.join("/build", self.args.name),
get_output=True).rstrip("\n")
else:
rev = (
self.args.git_path
if self.args.git_path is not None else "HEAD")
status["revision_id"] = self.run_build_command(
- ["git", "-C", self.args.name, "rev-parse", rev],
+ ["git", "rev-parse", rev],
+ cwd=os.path.join("/build", self.args.name),
get_output=True).rstrip("\n")
self.save_status(status)
=== modified file 'lpbuildd/target/tests/test_build_snap.py'
--- lpbuildd/target/tests/test_build_snap.py 2018-02-04 00:16:23 +0000
+++ lpbuildd/target/tests/test_build_snap.py 2018-03-14 14:06:41 +0000
@@ -62,10 +62,10 @@
class RanBuildCommand(RanCommand):
- def __init__(self, args, cwd="/build", **kwargs):
+ def __init__(self, args, **kwargs):
kwargs.setdefault("LANG", "C.UTF-8")
kwargs.setdefault("SHELL", "/bin/sh")
- super(RanBuildCommand, self).__init__(args, cwd=cwd, **kwargs)
+ super(RanBuildCommand, self).__init__(args, **kwargs)
class FakeRevisionID(FakeMethod):
@@ -181,9 +181,10 @@
build_snap.backend.run = FakeRevisionID("42")
build_snap.repo()
self.assertThat(build_snap.backend.run.calls, MatchesListwise([
- RanBuildCommand(["ls", "/build"]),
- RanBuildCommand(["bzr", "branch", "lp:foo", "test-snap"]),
- RanBuildCommand(["bzr", "revno", "test-snap"], get_output=True),
+ RanBuildCommand(
+ ["bzr", "branch", "lp:foo", "test-snap"], cwd="/build"),
+ RanBuildCommand(
+ ["bzr", "revno"], cwd="/build/test-snap", get_output=True),
]))
status_path = os.path.join(build_snap.backend.build_path, "status")
with open(status_path) as status:
@@ -200,13 +201,14 @@
build_snap.backend.run = FakeRevisionID("0" * 40)
build_snap.repo()
self.assertThat(build_snap.backend.run.calls, MatchesListwise([
- RanBuildCommand(["git", "clone", "lp:foo", "test-snap"]),
- RanBuildCommand(
- ["git", "-C", "test-snap",
- "submodule", "update", "--init", "--recursive"]),
- RanBuildCommand(
- ["git", "-C", "test-snap", "rev-parse", "HEAD"],
- get_output=True),
+ RanBuildCommand(
+ ["git", "clone", "lp:foo", "test-snap"], cwd="/build"),
+ RanBuildCommand(
+ ["git", "submodule", "update", "--init", "--recursive"],
+ cwd="/build/test-snap"),
+ RanBuildCommand(
+ ["git", "rev-parse", "HEAD"],
+ cwd="/build/test-snap", get_output=True),
]))
status_path = os.path.join(build_snap.backend.build_path, "status")
with open(status_path) as status:
@@ -224,13 +226,14 @@
build_snap.repo()
self.assertThat(build_snap.backend.run.calls, MatchesListwise([
RanBuildCommand(
- ["git", "clone", "-b", "next", "lp:foo", "test-snap"]),
- RanBuildCommand(
- ["git", "-C", "test-snap",
- "submodule", "update", "--init", "--recursive"]),
- RanBuildCommand(
- ["git", "-C", "test-snap", "rev-parse", "next"],
- get_output=True),
+ ["git", "clone", "-b", "next", "lp:foo", "test-snap"],
+ cwd="/build"),
+ RanBuildCommand(
+ ["git", "submodule", "update", "--init", "--recursive"],
+ cwd="/build/test-snap"),
+ RanBuildCommand(
+ ["git", "rev-parse", "next"],
+ cwd="/build/test-snap", get_output=True),
]))
status_path = os.path.join(build_snap.backend.build_path, "status")
with open(status_path) as status:
@@ -254,13 +257,14 @@
"GIT_PROXY_COMMAND": "/usr/local/bin/snap-git-proxy",
}
self.assertThat(build_snap.backend.run.calls, MatchesListwise([
- RanBuildCommand(["git", "clone", "lp:foo", "test-snap"], **env),
- RanBuildCommand(
- ["git", "-C", "test-snap",
- "submodule", "update", "--init", "--recursive"], **env),
- RanBuildCommand(
- ["git", "-C", "test-snap", "rev-parse", "HEAD"],
- get_output=True),
+ RanBuildCommand(
+ ["git", "clone", "lp:foo", "test-snap"], cwd="/build", **env),
+ RanBuildCommand(
+ ["git", "submodule", "update", "--init", "--recursive"],
+ cwd="/build/test-snap", **env),
+ RanBuildCommand(
+ ["git", "rev-parse", "HEAD"],
+ cwd="/build/test-snap", get_output=True),
]))
status_path = os.path.join(build_snap.backend.build_path, "status")
with open(status_path) as status:
@@ -355,7 +359,7 @@
self.assertThat(build_snap.backend.run.calls, MatchesAll(
AnyMatch(RanAptGet("install", "bzr", "snapcraft")),
AnyMatch(RanBuildCommand(
- ["bzr", "branch", "lp:foo", "test-snap"])),
+ ["bzr", "branch", "lp:foo", "test-snap"], cwd="/build")),
AnyMatch(RanBuildCommand(
["snapcraft", "pull"], cwd="/build/test-snap",
SNAPCRAFT_LOCAL_SOURCES="1", SNAPCRAFT_SETUP_CORE="1",
=== added file 'lpbuildd/target/vcs.py'
--- lpbuildd/target/vcs.py 1970-01-01 00:00:00 +0000
+++ lpbuildd/target/vcs.py 2018-03-14 14:06:41 +0000
@@ -0,0 +1,91 @@
+# Copyright 2018 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from __future__ import print_function
+
+__metaclass__ = type
+
+from collections import OrderedDict
+import logging
+import os.path
+import subprocess
+
+
+logger = logging.getLogger(__name__)
+
+
+class VCSOperationMixin:
+ """Methods supporting operations that check out a branch from a VCS."""
+
+ @classmethod
+ def add_arguments(cls, parser):
+ super(VCSOperationMixin, cls).add_arguments(parser)
+ build_from_group = parser.add_mutually_exclusive_group(required=True)
+ build_from_group.add_argument(
+ "--branch", metavar="BRANCH", help="build from this Bazaar branch")
+ build_from_group.add_argument(
+ "--git-repository", metavar="REPOSITORY",
+ help="build from this Git repository")
+ parser.add_argument(
+ "--git-path", metavar="REF-PATH",
+ help="build from this ref path in REPOSITORY")
+
+ def __init__(self, args, parser):
+ super(VCSOperationMixin, self).__init__(args, parser)
+ if args.git_repository is None and args.git_path is not None:
+ parser.error("--git-path requires --git-repository")
+ # Set to False for local testing if your target doesn't have an
+ # appropriate certificate for your codehosting system.
+ self.ssl_verify = True
+
+ @property
+ def vcs_description(self):
+ if self.args.branch is not None:
+ return self.args.branch
+ else:
+ assert self.args.git_repository is not None
+ description = self.args.git_repository
+ if self.args.git_path is not None:
+ description += " " + self.args.git_path
+ return description
+
+ @property
+ def vcs_deps(self):
+ if self.args.branch is not None:
+ return ["bzr"]
+ else:
+ return ["git"]
+
+ def vcs_fetch(self, name, cwd, env=None, quiet=False):
+ full_env = OrderedDict()
+ full_env["LANG"] = "C.UTF-8"
+ full_env["SHELL"] = "/bin/sh"
+ if env:
+ full_env.update(env)
+ if self.args.branch is not None:
+ cmd = ["bzr", "branch"]
+ if quiet:
+ cmd.append("-q")
+ cmd.extend([self.args.branch, name])
+ if not self.ssl_verify:
+ cmd.insert(1, "-Ossl.cert_reqs=none")
+ else:
+ assert self.args.git_repository is not None
+ cmd = ["git", "clone"]
+ if quiet:
+ cmd.append("-q")
+ if self.args.git_path is not None:
+ cmd.extend(["-b", self.args.git_path])
+ cmd.extend([self.args.git_repository, name])
+ if not self.ssl_verify:
+ env["GIT_SSL_NO_VERIFY"] = "1"
+ self.backend.run(cmd, cwd=cwd, env=full_env)
+ if self.args.git_repository is not None:
+ try:
+ self.backend.run(
+ ["git", "submodule", "update", "--init", "--recursive"],
+ cwd=os.path.join(cwd, name), env=full_env)
+ except subprocess.CalledProcessError as e:
+ logger.error(
+ "'git submodule update --init --recursive failed with "
+ "exit code %s (build may fail later)" % e.returncode)
