launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #29208
[Merge] ~cjwatson/launchpad-buildd:ci-clamav into launchpad-buildd:master
Colin Watson has proposed merging ~cjwatson/launchpad-buildd:ci-clamav into launchpad-buildd:master.
Commit message:
Add optional malware scanning at the end of CI build jobs
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad-buildd/+git/launchpad-buildd/+merge/430040
This is currently implemented using clamav. It's probably not yet amazingly effective, and I expect we'd need to start doing on-access scanning in order to get much better, but it gives us a starting point.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad-buildd:ci-clamav into launchpad-buildd:master.
diff --git a/debian/changelog b/debian/changelog
index aa2253a..0d35fce 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,8 @@
launchpad-buildd (222) UNRELEASED; urgency=medium
* Remove use of six.
+ * Add optional malware scanning at the end of CI build jobs, currently
+ implemented using clamav.
-- Colin Watson <cjwatson@xxxxxxxxxx> Mon, 12 Sep 2022 09:50:13 +0100
diff --git a/lpbuildd/ci.py b/lpbuildd/ci.py
index c49a24c..6083298 100644
--- a/lpbuildd/ci.py
+++ b/lpbuildd/ci.py
@@ -64,6 +64,7 @@ class CIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
self.environment_variables = extra_args.get("environment_variables")
self.plugin_settings = extra_args.get("plugin_settings")
self.secrets = extra_args.get("secrets")
+ self.scan_malware = extra_args.get("scan_malware", False)
super().initiate(files, chroot, extra_args)
@@ -82,6 +83,8 @@ class CIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
args.extend(["--git-repository", self.git_repository])
if self.git_path is not None:
args.extend(["--git-path", self.git_path])
+ if self.scan_malware:
+ args.append("--scan-malware")
try:
snap_store_proxy_url = self._builder._config.get(
"proxy", "snapstore")
@@ -164,6 +167,8 @@ class CIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
)
args.extend(
["--secrets", "/build/.launchpad-secrets.yaml"])
+ if self.scan_malware:
+ args.append("--scan-malware")
job_name, job_index = self.current_job
self.current_job_id = _make_job_id(job_name, job_index)
diff --git a/lpbuildd/target/run_ci.py b/lpbuildd/target/run_ci.py
index 8f23b01..491943b 100644
--- a/lpbuildd/target/run_ci.py
+++ b/lpbuildd/target/run_ci.py
@@ -31,6 +31,12 @@ class RunCIPrepare(BuilderProxyOperationMixin, VCSOperationMixin,
parser.add_argument(
"--channel", action=SnapChannelsAction, metavar="SNAP=CHANNEL",
dest="channels", default={}, help="install SNAP from CHANNEL")
+ parser.add_argument(
+ "--scan-malware",
+ action="store_true",
+ default=False,
+ help="perform malware scans on output files",
+ )
def install(self):
logger.info("Running install phase...")
@@ -43,6 +49,8 @@ class RunCIPrepare(BuilderProxyOperationMixin, VCSOperationMixin,
if self.backend.is_package_available(dep):
deps.append(dep)
deps.extend(self.vcs_deps)
+ if self.args.scan_malware:
+ deps.append("clamav")
self.backend.run(["apt-get", "-y", "install"] + deps)
if self.backend.supports_snapd:
self.snap_store_set_proxy()
@@ -59,6 +67,16 @@ class RunCIPrepare(BuilderProxyOperationMixin, VCSOperationMixin,
cmd.append(snap_name)
self.backend.run(cmd)
self.backend.run(["lxd", "init", "--auto"])
+ if self.args.scan_malware:
+ # lpbuildd.target.lxd configures the container not to run most
+ # services, which is convenient since it allows us to ensure
+ # that ClamAV's database is up to date before proceeding.
+ kwargs = {}
+ env = self.build_proxy_environment(proxy_url=self.args.proxy_url)
+ if env:
+ kwargs["env"] = env
+ logger.info("Downloading malware definitions...")
+ self.backend.run(["freshclam", "--quiet"], **kwargs)
def repo(self):
"""Collect VCS branch."""
@@ -121,6 +139,12 @@ class RunCI(BuilderProxyOperationMixin, Operation):
type=str,
help="secrets where the key and the value are separated by =",
)
+ parser.add_argument(
+ "--scan-malware",
+ action="store_true",
+ default=False,
+ help="perform malware scans on output files",
+ )
def run_job(self):
logger.info("Running job phase...")
@@ -172,6 +196,10 @@ class RunCI(BuilderProxyOperationMixin, Operation):
]
self.run_build_command(args, env=env)
+ if self.args.scan_malware:
+ clamscan = ["clamscan", "--recursive", job_output_path]
+ self.run_build_command(clamscan, env=env)
+
def run(self):
try:
self.run_job()
diff --git a/lpbuildd/target/tests/test_run_ci.py b/lpbuildd/target/tests/test_run_ci.py
index 0b0b77b..ba941ea 100644
--- a/lpbuildd/target/tests/test_run_ci.py
+++ b/lpbuildd/target/tests/test_run_ci.py
@@ -141,6 +141,53 @@ class TestRunCIPrepare(TestCase):
RanCommand(["lxd", "init", "--auto"]),
]))
+ def test_install_scan_malware(self):
+ args = [
+ "run-ci-prepare",
+ "--backend=fake", "--series=focal", "--arch=amd64", "1",
+ "--git-repository", "lp:foo",
+ "--scan-malware",
+ ]
+ run_ci_prepare = parse_args(args=args).operation
+ run_ci_prepare.install()
+ self.assertThat(run_ci_prepare.backend.run.calls, MatchesListwise([
+ RanAptGet("install", "git", "clamav"),
+ RanSnap("install", "lxd"),
+ RanSnap("install", "--classic", "lpcraft"),
+ RanCommand(["lxd", "init", "--auto"]),
+ RanCommand(["freshclam", "--quiet"]),
+ ]))
+
+ def test_install_scan_malware_proxy(self):
+ args = [
+ "run-ci-prepare",
+ "--backend=fake", "--series=focal", "--arch=amd64", "1",
+ "--git-repository", "lp:foo",
+ "--proxy-url", "http://proxy.example:3128/";,
+ "--scan-malware",
+ ]
+ run_ci_prepare = parse_args(args=args).operation
+ run_ci_prepare.bin = "/builderbin"
+ self.useFixture(FakeFilesystem()).add("/builderbin")
+ os.mkdir("/builderbin")
+ with open("/builderbin/lpbuildd-git-proxy", "w") as proxy_script:
+ proxy_script.write("proxy script\n")
+ os.fchmod(proxy_script.fileno(), 0o755)
+ run_ci_prepare.install()
+ env = {
+ "http_proxy": "http://proxy.example:3128/";,
+ "https_proxy": "http://proxy.example:3128/";,
+ "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy",
+ "SNAPPY_STORE_NO_CDN": "1",
+ }
+ self.assertThat(run_ci_prepare.backend.run.calls, MatchesListwise([
+ RanAptGet("install", "python3", "socat", "git", "clamav"),
+ RanSnap("install", "lxd"),
+ RanSnap("install", "--classic", "lpcraft"),
+ RanCommand(["lxd", "init", "--auto"]),
+ RanCommand(["freshclam", "--quiet"], **env),
+ ]))
+
def test_repo_git(self):
args = [
"run-ci-prepare",
@@ -440,6 +487,47 @@ class TestRunCI(TestCase):
], cwd="/build/tree"),
]))
+ def test_run_job_scan_malware_succeeds(self):
+ args = [
+ "run-ci",
+ "--backend=fake", "--series=focal", "--arch=amd64", "1",
+ "--scan-malware",
+ "test", "0",
+ ]
+ run_ci = parse_args(args=args).operation
+ run_ci.run_job()
+ self.assertThat(run_ci.backend.run.calls, MatchesListwise([
+ RanCommand(["mkdir", "-p", "/build/output/test/0"]),
+ RanBuildCommand([
+ "/bin/bash", "-o", "pipefail", "-c",
+ "lpcraft -v run-one --output-directory /build/output "
+ "test 0 "
+ "2>&1 "
+ "| tee /build/output/test/0/log",
+ ], cwd="/build/tree"),
+ RanBuildCommand(
+ ["clamscan", "--recursive", "/build/output/test/0"],
+ cwd="/build/tree"),
+ ]))
+
+ def test_run_job_scan_malware_fails(self):
+ class FailClamscan(FakeMethod):
+ def __call__(self, run_args, *args, **kwargs):
+ super().__call__(run_args, *args, **kwargs)
+ if run_args[0] == "clamscan":
+ raise subprocess.CalledProcessError(1, run_args)
+
+ self.useFixture(FakeLogger())
+ args = [
+ "run-ci",
+ "--backend=fake", "--series=focal", "--arch=amd64", "1",
+ "--scan-malware",
+ "test", "0",
+ ]
+ run_ci = parse_args(args=args).operation
+ run_ci.backend.run = FailClamscan()
+ self.assertRaises(subprocess.CalledProcessError, run_ci.run_job)
+
def test_run_succeeds(self):
args = [
"run-ci",
diff --git a/lpbuildd/tests/test_ci.py b/lpbuildd/tests/test_ci.py
index 88dda19..bf1b468 100644
--- a/lpbuildd/tests/test_ci.py
+++ b/lpbuildd/tests/test_ci.py
@@ -128,11 +128,13 @@ class TestCIBuildManagerIteration(TestCase):
},
"secrets": {
"auth": "user:pass",
- }
+ },
+ "scan_malware": True,
}
expected_prepare_options = [
"--git-repository", "https://git.launchpad.test/~example/+git/ci";,
"--git-path", "main",
+ "--scan-malware",
]
yield self.startBuild(args, expected_prepare_options)
@@ -145,6 +147,7 @@ class TestCIBuildManagerIteration(TestCase):
"--plugin-setting", "miniconda_conda_channel=https://user:pass@xxxxxxxxxxxxxxxxxxxxx/artifactory/soss-conda-stable-local/";, # noqa: E501
"--plugin-setting", "foo=bar",
"--secrets", "/build/.launchpad-secrets.yaml",
+ "--scan-malware",
]
yield self.expectRunJob("build", "0", options=expected_job_options)
self.buildmanager.backend.add_file(
