launchpad-reviewers team mailing list archive

[Colin Watson <mp+432172@xxxxxxxxxxxxxxxxxx>]

Colin Watson has proposed merging ~cjwatson/launchpad-buildd:clamav-database-url into launchpad-buildd:master.

Commit message:
Allow configuring builders to use a different ClamAV database URL

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~cjwatson/launchpad-buildd/+git/launchpad-buildd/+merge/432172

This is a property of the environment rather than of the build, so we put it in the builder's configuration file.
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad-buildd:clamav-database-url into launchpad-buildd:master.

diff --git a/debian/changelog b/debian/changelog
index dab52b4..41fce54 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+launchpad-buildd (224) UNRELEASED; urgency=medium
+
+  * Allow configuring builders to use a different ClamAV database URL.
+
+ -- Colin Watson <cjwatson@xxxxxxxxxx>  Wed, 26 Oct 2022 08:55:39 +0200
+
 launchpad-buildd (223) focal; urgency=medium
 
   * Add optional malware scanning at the end of CI build jobs, currently
diff --git a/lpbuildd/ci.py b/lpbuildd/ci.py
index 6083298..04aa5f8 100644
--- a/lpbuildd/ci.py
+++ b/lpbuildd/ci.py
@@ -86,6 +86,17 @@ class CIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
         if self.scan_malware:
             args.append("--scan-malware")
         try:
+            # Not precisely a proxy, but it's similar in the sense of
+            # providing additional network endpoints that we use instead of
+            # the default behaviour, and using a section that doesn't exist
+            # in the default configuration is convenient for our production
+            # deployments.
+            clamav_database_url = self._builder._config.get(
+                "proxy", "clamavdatabase")
+            args.extend(["--clamav-database-url", clamav_database_url])
+        except (NoSectionError, NoOptionError):
+            pass
+        try:
             snap_store_proxy_url = self._builder._config.get(
                 "proxy", "snapstore")
             args.extend(["--snap-store-proxy-url", snap_store_proxy_url])
diff --git a/lpbuildd/target/run_ci.py b/lpbuildd/target/run_ci.py
index 491943b..4ebf765 100644
--- a/lpbuildd/target/run_ci.py
+++ b/lpbuildd/target/run_ci.py
@@ -3,6 +3,7 @@
 
 import logging
 import os
+import tempfile
 
 from lpbuildd.target.build_snap import SnapChannelsAction
 from lpbuildd.target.operation import Operation
@@ -37,6 +38,10 @@ class RunCIPrepare(BuilderProxyOperationMixin, VCSOperationMixin,
             default=False,
             help="perform malware scans on output files",
         )
+        parser.add_argument(
+            "--clamav-database-url",
+            help="override default ClamAV database URL",
+        )
 
     def install(self):
         logger.info("Running install phase...")
@@ -71,6 +76,17 @@ class RunCIPrepare(BuilderProxyOperationMixin, VCSOperationMixin,
             # lpbuildd.target.lxd configures the container not to run most
             # services, which is convenient since it allows us to ensure
             # that ClamAV's database is up to date before proceeding.
+            if self.args.clamav_database_url:
+                freshclam_path = "/etc/clamav/freshclam.conf"
+                with tempfile.NamedTemporaryFile(mode="w+") as freshclam_file:
+                    self.backend.copy_out(freshclam_path, freshclam_file.name)
+                    freshclam_file.seek(0, os.SEEK_END)
+                    print(
+                        f"PrivateMirror {self.args.clamav_database_url}",
+                        file=freshclam_file,
+                    )
+                    freshclam_file.flush()
+                    self.backend.copy_in(freshclam_file.name, freshclam_path)
             kwargs = {}
             env = self.build_proxy_environment(proxy_url=self.args.proxy_url)
             if env:
diff --git a/lpbuildd/target/tests/test_run_ci.py b/lpbuildd/target/tests/test_run_ci.py
index ba941ea..c855eec 100644
--- a/lpbuildd/target/tests/test_run_ci.py
+++ b/lpbuildd/target/tests/test_run_ci.py
@@ -188,6 +188,30 @@ class TestRunCIPrepare(TestCase):
             RanCommand(["freshclam", "--quiet"], **env),
             ]))
 
+    def test_install_scan_malware_with_clamav_database_url(self):
+        args = [
+            "run-ci-prepare",
+            "--backend=fake", "--series=focal", "--arch=amd64", "1",
+            "--git-repository", "lp:foo",
+            "--scan-malware",
+            "--clamav-database-url", "http://clamav.example/";,
+            ]
+        run_ci_prepare = parse_args(args=args).operation
+        run_ci_prepare.backend.add_file(
+            "/etc/clamav/freshclam.conf", b"Test line\n")
+        run_ci_prepare.install()
+        self.assertThat(run_ci_prepare.backend.run.calls, MatchesListwise([
+            RanAptGet("install", "git", "clamav"),
+            RanSnap("install", "lxd"),
+            RanSnap("install", "--classic", "lpcraft"),
+            RanCommand(["lxd", "init", "--auto"]),
+            RanCommand(["freshclam", "--quiet"]),
+            ]))
+        self.assertEqual(
+            (b"Test line\nPrivateMirror http://clamav.example/\n";,
+             stat.S_IFREG | 0o644),
+            run_ci_prepare.backend.backend_fs["/etc/clamav/freshclam.conf"])
+
     def test_repo_git(self):
         args = [
             "run-ci-prepare",
diff --git a/lpbuildd/tests/test_ci.py b/lpbuildd/tests/test_ci.py
index bf1b468..d8a9ef9 100644
--- a/lpbuildd/tests/test_ci.py
+++ b/lpbuildd/tests/test_ci.py
@@ -379,3 +379,23 @@ class TestCIBuildManagerIteration(TestCase):
         shutil.rmtree(get_build_path(
             self.buildmanager.home, self.buildmanager._buildid))
         self.assertIn("jobs", self.buildmanager.status())
+
+    @defer.inlineCallbacks
+    def test_iterate_with_clamav_database_url(self):
+        # If proxy.clamavdatabase is set, the build manager passes it via
+        # the --clamav-database-url option.
+        self.builder._config.set(
+            "proxy", "clamavdatabase", "http://clamav.example/";)
+        args = {
+            "git_repository": "https://git.launchpad.test/~example/+git/ci";,
+            "git_path": "main",
+            "jobs": [[("build", "0")], [("test", "0")]],
+            "scan_malware": True,
+        }
+        expected_prepare_options = [
+            "--git-repository", "https://git.launchpad.test/~example/+git/ci";,
+            "--git-path", "main",
+            "--scan-malware",
+            "--clamav-database-url", "http://clamav.example/";,
+            ]
+        yield self.startBuild(args, expected_prepare_options)