launchpad-reviewers team mailing list archive

[Guruprasad <mp+439809@xxxxxxxxxxxxxxxxxx>]

Guruprasad has proposed merging ~lgp171188/launchpad:rsync-oval-data-when-publishing-a-distro into launchpad:master.

Commit message:
Perform OVAL data rsync when publishing a distro

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~lgp171188/launchpad/+git/launchpad/+merge/439809
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~lgp171188/launchpad:rsync-oval-data-when-publishing-a-distro into launchpad:master.

diff --git a/lib/lp/archivepublisher/scripts/publishdistro.py b/lib/lp/archivepublisher/scripts/publishdistro.py
index 3b91fa5..82068ee 100644
--- a/lib/lp/archivepublisher/scripts/publishdistro.py
+++ b/lib/lp/archivepublisher/scripts/publishdistro.py
@@ -8,6 +8,7 @@ __all__ = [
 ]
 
 from optparse import OptionValueError
+from subprocess import CalledProcessError, check_call
 
 from storm.store import Store
 from zope.component import getUtility
@@ -19,6 +20,7 @@ from lp.archivepublisher.publishing import (
     getPublisher,
 )
 from lp.archivepublisher.scripts.base import PublisherScript
+from lp.services.config import config
 from lp.services.limitedlist import LimitedList
 from lp.services.scripts.base import LaunchpadScriptFailure
 from lp.services.webapp.adapter import (
@@ -504,11 +506,49 @@ class PublishDistro(PublisherScript):
                 # store and cause performance problems.
                 Store.of(archive).reset()
 
+    def rsync_oval_data(self):
+        if config.archivepublisher.oval_data_rsync_endpoint:
+            rsync_src = config.archivepublisher.oval_data_rsync_endpoint
+            rsync_dest = config.archivepublisher.oval_data_rsync_destination
+            rsync_command = [
+                "/usr/bin/rsync",
+                "-a",
+                "-q",
+                "--timeout={}".format(
+                    config.archivepublisher.oval_data_rsync_timeout
+                ),
+                "--delete",
+                "--delete-after",
+                rsync_src,
+                rsync_dest,
+            ]
+            try:
+                self.logger.info(
+                    "Attempting to rsync the OVAL data from '%s' to '%s'",
+                    rsync_src,
+                    rsync_dest,
+                )
+                check_call(rsync_command)
+            except CalledProcessError:
+                self.logger.exception(
+                    "Failed to rsync OVAL data from '%s' to '%s'",
+                    rsync_src,
+                    rsync_dest,
+                )
+                raise
+        else:
+            self.logger.info(
+                "Skipping the OVAL data sync as no rsync endpoint"
+                " has been configured."
+            )
+
     def main(self, reset_store_between_archives=True):
         """See `LaunchpadScript`."""
         self.validateOptions()
         self.logOptions()
 
+        self.rsync_oval_data()
+
         archive_ids = []
         for distribution in self.findDistros():
             for archive in self.getTargetArchives(distribution):
diff --git a/lib/lp/archivepublisher/tests/test_publishdistro.py b/lib/lp/archivepublisher/tests/test_publishdistro.py
index fd1d1e6..96f77b0 100644
--- a/lib/lp/archivepublisher/tests/test_publishdistro.py
+++ b/lib/lp/archivepublisher/tests/test_publishdistro.py
@@ -8,6 +8,7 @@ import shutil
 import subprocess
 from optparse import OptionValueError
 
+from fixtures import MockPatch
 from storm.store import Store
 from testtools.matchers import Not, PathExists
 from testtools.twistedsupport import AsynchronousDeferredRunTest
@@ -255,6 +256,73 @@ class TestPublishDistro(TestNativePublishingBase):
         pub_source = self.loadPubSource(pub_source_id)
         self.assertEqual(PackagePublishingStatus.PENDING, pub_source.status)
 
+    def setUpOVALDataRsync(self):
+        config.push(
+            "oval-data-rsync",
+            """
+            [archivepublisher]
+            oval_data_rsync_endpoint: oval.internal::oval/
+            oval_data_rsync_destination: /tmp/oval-data
+            oval_data_rsync_timeout: 90
+            """,
+        )
+        self.addCleanup(config.pop, "oval-data-rsync")
+
+    def testPublishDistroOVALDataRsyncEndpointNotConfigured(self):
+        """
+        Test what happens when the OVAL data rsync endpoint is not configured.
+        """
+        mock_subprocess_check_call = self.useFixture(
+            MockPatch("lp.archivepublisher.scripts.publishdistro.check_call")
+        ).mock
+        self.runPublishDistro()
+        mock_subprocess_check_call.assert_not_called()
+
+    def testPublishDistroOVALDataRsyncEndpointConfigured(self):
+        """
+        Test the OVAL data rsync command.
+
+        When the endpoint is configured, verify that the command is run.
+        """
+        self.setUpOVALDataRsync()
+        mock_subprocess_check_call = self.useFixture(
+            MockPatch("lp.archivepublisher.scripts.publishdistro.check_call")
+        ).mock
+        self.runPublishDistro()
+        call_args = [
+            "/usr/bin/rsync",
+            "-a",
+            "-q",
+            "--timeout=90",
+            "--delete",
+            "--delete-after",
+            "oval.internal::oval/",
+            "/tmp/oval-data",
+        ]
+        mock_subprocess_check_call.assert_called_with(call_args)
+
+    def testPublishDistroOVALDataRsyncErrorsOut(self):
+        self.setUpOVALDataRsync()
+        mock_subprocess_check_call = self.useFixture(
+            MockPatch(
+                "lp.archivepublisher.scripts.publishdistro.check_call",
+                side_effect=subprocess.CalledProcessError(
+                    cmd="foo", returncode=5
+                ),
+            )
+        ).mock
+        call_args = [
+            "/usr/bin/rsync",
+            "-a",
+            "-q",
+            "--timeout=90",
+            "--delete",
+            "--delete-after",
+            "oval.internal::oval/",
+            "/tmp/oval-data",
+        ]
+        mock_subprocess_check_call.assert_called_with(call_args)
+
     @defer.inlineCallbacks
     def testForPPA(self):
         """Try to run publish-distro in PPA mode.
diff --git a/lib/lp/services/config/schema-lazr.conf b/lib/lp/services/config/schema-lazr.conf
index db9893c..319c4cb 100644
--- a/lib/lp/services/config/schema-lazr.conf
+++ b/lib/lp/services/config/schema-lazr.conf
@@ -32,6 +32,16 @@ dbuser: archivepublisher
 # datatype: string
 run_parts_location: none
 
+# The rsync endpoint to use for synchronizing the OVAL data.
+oval_data_rsync_endpoint: none
+
+# The destination to rsync the OVAL data to.
+oval_data_rsync_destination: /srv/launchpad.net/oval-data
+
+# Timeout (in seconds) to use when rsync'ing OVAL data.
+oval_data_rsync_timeout: 30
+
+
 
 [artifactory]
 # Base URL for publishing suitably-configured archives to Artifactory.