launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #32885
[Merge] ~ruinedyourlife/launchpad-buildd:clamav-craft-builds into launchpad-buildd:master
RuinedYourLife has proposed merging ~ruinedyourlife/launchpad-buildd:clamav-craft-builds into launchpad-buildd:master.
Commit message:
Malware scanning for craft builds
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~ruinedyourlife/launchpad-buildd/+git/launchpad-buildd/+merge/491153
This is a copy paste from the existing implementation for ci builds:
`lpbuildd/ci.py`
`lpbuildd/tests/test_ci.py`
`lpbuildd/target/run_ci.py`
`lpbuildd/target/tests/test_run_ci.py`
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~ruinedyourlife/launchpad-buildd:clamav-craft-builds into launchpad-buildd:master.
diff --git a/lpbuildd/craft.py b/lpbuildd/craft.py
index d61ef34..62d78a3 100644
--- a/lpbuildd/craft.py
+++ b/lpbuildd/craft.py
@@ -39,6 +39,7 @@ class CraftBuildManager(BuildManagerProxyMixin, DebianBuildManager):
self.launchpad_server_url = extra_args.get("launchpad_server_url")
self.proxy_service = None
self.environment_variables = extra_args.get("environment_variables")
+ self.scan_malware = extra_args.get("scan_malware", False)
super().initiate(files, chroot, extra_args)
@@ -73,6 +74,16 @@ class CraftBuildManager(BuildManagerProxyMixin, DebianBuildManager):
if self.environment_variables:
for key, value in self.environment_variables.items():
args.extend(["--environment-variable", f"{key}={value}"])
+ if self.scan_malware:
+ args.append("--scan-malware")
+ # Optional ClamAV DB override via builder config (same pattern as CI)
+ try:
+ clamav_database_url = self._builder._config.get(
+ "proxy", "clamavdatabase"
+ )
+ args.extend(["--clamav-database-url", clamav_database_url])
+ except Exception:
+ pass
args.append(self.name)
self.runTargetSubProcess("build-craft", *args)
diff --git a/lpbuildd/target/build_craft.py b/lpbuildd/target/build_craft.py
index 81cb27f..6998740 100644
--- a/lpbuildd/target/build_craft.py
+++ b/lpbuildd/target/build_craft.py
@@ -74,6 +74,16 @@ class BuildCraft(
type=str,
help="launchpad server url.",
)
+ parser.add_argument(
+ "--scan-malware",
+ action="store_true",
+ default=False,
+ help="perform malware scans on output files",
+ )
+ parser.add_argument(
+ "--clamav-database-url",
+ help="override default ClamAV database URL",
+ )
def __init__(self, args, parser):
super().__init__(args, parser)
@@ -95,6 +105,8 @@ class BuildCraft(
if self.backend.is_package_available(dep):
deps.append(dep)
deps.extend(self.vcs_deps)
+ if self.args.scan_malware:
+ deps.append("clamav")
# See charmcraft.provider.CharmcraftBuilddBaseConfiguration.setup.
self.backend.run(["apt-get", "-y", "install"] + deps)
if self.backend.supports_snapd:
@@ -141,6 +153,24 @@ class BuildCraft(
# We could build the craft in /build, but we are using /home/buildd
# for consistency with other build types.
self.backend.run(["mkdir", "-p", "/home/buildd"])
+ if self.args.scan_malware:
+ # Ensure ClamAV database is up to date before any scans.
+ if self.args.clamav_database_url:
+ with self.backend.open(
+ "/etc/clamav/freshclam.conf", mode="a"
+ ) as freshclam_file:
+ freshclam_file.write(
+ f"PrivateMirror {self.args.clamav_database_url}\n"
+ )
+ kwargs = {}
+ env = self.build_proxy_environment(
+ proxy_url=self.args.proxy_url,
+ use_fetch_service=self.args.use_fetch_service,
+ )
+ if env:
+ kwargs["env"] = env
+ logger.info("Downloading malware definitions...")
+ self.backend.run(["freshclam", "--quiet"], **kwargs)
def repo(self):
"""Collect git or bzr branch."""
@@ -381,6 +411,14 @@ class BuildCraft(
args = ["sourcecraft", "pack", "-v", "--destructive-mode"]
self.run_build_command(args, env=env, cwd=build_context_path)
+ if self.args.scan_malware:
+ # Scan the output directory for malware after building.
+ output_path = os.path.join("/home/buildd", self.args.name)
+ if self.args.build_path is not None:
+ output_path = os.path.join(output_path, self.args.build_path)
+ clamscan = ["clamscan", "--recursive", output_path]
+ self.run_build_command(clamscan, env=env)
+
def run(self):
try:
self.install()
diff --git a/lpbuildd/target/tests/test_build_craft.py b/lpbuildd/target/tests/test_build_craft.py
index 98f2623..839b432 100644
--- a/lpbuildd/target/tests/test_build_craft.py
+++ b/lpbuildd/target/tests/test_build_craft.py
@@ -1399,3 +1399,195 @@ class TestBuildCraft(TestCase):
]
),
)
+
+ def test_install_scan_malware(self):
+ args = [
+ "build-craft",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--git-repository",
+ "lp:foo",
+ "--scan-malware",
+ "test-image",
+ ]
+ build_craft = parse_args(args=args).operation
+ build_craft.install()
+ self.assertThat(
+ build_craft.backend.run.calls,
+ MatchesListwise(
+ [
+ RanAptGet("install", "git", "clamav"),
+ RanSnap(
+ "install",
+ "--classic",
+ "--channel=latest/edge",
+ "sourcecraft",
+ ),
+ RanCommand(["mkdir", "-p", "/home/buildd"]),
+ RanCommand(["freshclam", "--quiet"]),
+ ]
+ ),
+ )
+
+ def test_install_scan_malware_proxy(self):
+ args = [
+ "build-craft",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--git-repository",
+ "lp:foo",
+ "--proxy-url",
+ "http://proxy.example:3128/";,
+ "--scan-malware",
+ "test-image",
+ ]
+ build_craft = parse_args(args=args).operation
+ build_craft.bin = "/builderbin"
+ self.useFixture(FakeFilesystem()).add("/builderbin")
+ os.mkdir("/builderbin")
+ with open("/builderbin/lpbuildd-git-proxy", "w") as proxy_script:
+ proxy_script.write("proxy script\n")
+ os.fchmod(proxy_script.fileno(), 0o755)
+ build_craft.install()
+ env = {
+ "http_proxy": "http://proxy.example:3128/";,
+ "https_proxy": "http://proxy.example:3128/";,
+ "HTTP_PROXY": "http://proxy.example:3128/";,
+ "HTTPS_PROXY": "http://proxy.example:3128/";,
+ "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy",
+ "SNAPPY_STORE_NO_CDN": "1",
+ }
+ self.assertThat(
+ build_craft.backend.run.calls,
+ MatchesListwise(
+ [
+ RanAptGet(
+ "install",
+ "python3",
+ "socat",
+ "git",
+ "clamav",
+ ),
+ RanSnap(
+ "install",
+ "--classic",
+ "--channel=latest/edge",
+ "sourcecraft",
+ ),
+ RanCommand(["mkdir", "-p", "/home/buildd"]),
+ RanCommand(["freshclam", "--quiet"], **env),
+ ]
+ ),
+ )
+ self.assertEqual(
+ (b"proxy script\n", stat.S_IFREG | 0o755),
+ build_craft.backend.backend_fs[
+ "/usr/local/bin/lpbuildd-git-proxy"
+ ],
+ )
+
+ def test_install_scan_malware_with_clamav_database_url(self):
+ args = [
+ "build-craft",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--git-repository",
+ "lp:foo",
+ "--scan-malware",
+ "--clamav-database-url",
+ "http://clamav.example/";,
+ "test-image",
+ ]
+ build_craft = parse_args(args=args).operation
+ build_craft.backend.add_file(
+ "/etc/clamav/freshclam.conf", b"Test line\n"
+ )
+ build_craft.install()
+ self.assertThat(
+ build_craft.backend.run.calls,
+ MatchesListwise(
+ [
+ RanAptGet("install", "git", "clamav"),
+ RanSnap(
+ "install",
+ "--classic",
+ "--channel=latest/edge",
+ "sourcecraft",
+ ),
+ RanCommand(["mkdir", "-p", "/home/buildd"]),
+ RanCommand(["freshclam", "--quiet"]),
+ ]
+ ),
+ )
+ self.assertEqual(
+ (
+ b"Test line\nPrivateMirror http://clamav.example/\n";,
+ stat.S_IFREG | 0o644,
+ ),
+ build_craft.backend.backend_fs["/etc/clamav/freshclam.conf"],
+ )
+
+ def test_build_scan_malware_succeeds(self):
+ args = [
+ "build-craft",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--branch",
+ "lp:foo",
+ "--scan-malware",
+ "test-image",
+ ]
+ build_craft = parse_args(args=args).operation
+ build_craft.backend.add_dir("/build/test-directory")
+ build_craft.build()
+ self.assertThat(
+ build_craft.backend.run.calls,
+ MatchesListwise(
+ [
+ RanBuildCommand(
+ ["sourcecraft", "pack", "-v", "--destructive-mode"],
+ cwd="/home/buildd/test-image/.",
+ ),
+ RanBuildCommand(
+ [
+ "clamscan",
+ "--recursive",
+ "/home/buildd/test-image/.",
+ ],
+ cwd="/home/buildd/test-image",
+ ),
+ ]
+ ),
+ )
+
+ def test_run_scan_malware_fails(self):
+ class FailClamscan(FakeMethod):
+ def __call__(self, run_args, *args, **kwargs):
+ super().__call__(run_args, *args, **kwargs)
+ if run_args[0] == "clamscan":
+ raise subprocess.CalledProcessError(1, run_args)
+
+ self.useFixture(FakeLogger())
+ args = [
+ "build-craft",
+ "--backend=fake",
+ "--series=xenial",
+ "--arch=amd64",
+ "1",
+ "--branch",
+ "lp:foo",
+ "--scan-malware",
+ "test-image",
+ ]
+ build_craft = parse_args(args=args).operation
+ build_craft.backend.build_path = self.useFixture(TempDir()).path
+ build_craft.backend.run = FailClamscan()
+ self.assertEqual(RETCODE_FAILURE_BUILD, build_craft.run())
diff --git a/lpbuildd/tests/test_craft.py b/lpbuildd/tests/test_craft.py
index b9dfa40..fd38c3e 100644
--- a/lpbuildd/tests/test_craft.py
+++ b/lpbuildd/tests/test_craft.py
@@ -269,3 +269,41 @@ class TestCraftBuildManagerIteration(TestCase):
"launchpad.test",
]
yield self.startBuild(args, expected_options)
+
+ @defer.inlineCallbacks
+ def test_iterate_with_scan_malware(self):
+ # The build manager passes --scan-malware to subprocesses.
+ args = {
+ "git_repository": "https://git.launchpad.dev/~example/+git/craft";,
+ "git_path": "master",
+ "scan_malware": True,
+ }
+ expected_options = [
+ "--git-repository",
+ "https://git.launchpad.dev/~example/+git/craft";,
+ "--git-path",
+ "master",
+ "--scan-malware",
+ ]
+ yield self.startBuild(args, expected_options)
+
+ @defer.inlineCallbacks
+ def test_iterate_with_clamav_database_url(self):
+ # If proxy.clamavdatabase is set, the manager passes it via
+ # the --clamav-database-url option.
+ self.buildmanager._builder._config.set(
+ "proxy", "clamavdatabase", "http://clamav.example/";
+ )
+ args = {
+ "git_repository": "https://git.launchpad.dev/~example/+git/craft";,
+ "git_path": "master",
+ }
+ expected_options = [
+ "--git-repository",
+ "https://git.launchpad.dev/~example/+git/craft";,
+ "--git-path",
+ "master",
+ "--clamav-database-url",
+ "http://clamav.example/";,
+ ]
+ yield self.startBuild(args, expected_options)
