launchpad-reviewers team mailing list archive

[William Grant <me@xxxxxxxxxxxxxxxxxx>]

William Grant has proposed merging lp:~wgrant/launchpad/b-m-integration-test into lp:launchpad.

Commit message:
Add an end-to-end integration test for buildd-manager's SlaveScanner, testing from candidate acquisition to post-build cleanup, mocking out just the clock and slave.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~wgrant/launchpad/b-m-integration-test/+merge/222308

Add an end-to-end integration test for buildd-manager's SlaveScanner, testing from candidate acquisition to post-build cleanup, mocking out just the clock and slave.

I believe this is the most complete test of core buildd-manager functionality that we have.
-- 
https://code.launchpad.net/~wgrant/launchpad/b-m-integration-test/+merge/222308
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~wgrant/launchpad/b-m-integration-test into lp:launchpad.

=== modified file 'lib/lp/buildmaster/tests/mock_slaves.py'
--- lib/lp/buildmaster/tests/mock_slaves.py	2014-05-10 18:40:36 +0000
+++ lib/lp/buildmaster/tests/mock_slaves.py	2014-06-06 11:05:48 +0000
@@ -76,7 +76,12 @@
         self.arch_tag = arch_tag
         self.version = version
 
+    @property
+    def method_log(self):
+        return [(x[0] if isinstance(x, tuple) else x) for x in self.call_log]
+
     def status(self):
+        self.call_log.append('status')
         slave_status = {'builder_status': 'BuilderStatus.IDLE'}
         if self.version is not None:
             slave_status['builder_version'] = self.version
@@ -138,10 +143,13 @@
     def __init__(self, build_id='1-1'):
         super(BuildingSlave, self).__init__()
         self.build_id = build_id
+        self.status_count = 0
 
     def status(self):
         self.call_log.append('status')
-        buildlog = xmlrpclib.Binary("This is a build log")
+        buildlog = xmlrpclib.Binary(
+            "This is a build log: %d" % self.status_count)
+        self.status_count += 1
         return defer.succeed({
             'builder_status': 'BuilderStatus.BUILDING',
             'build_id': self.build_id,

=== modified file 'lib/lp/buildmaster/tests/test_manager.py'
--- lib/lp/buildmaster/tests/test_manager.py	2014-02-05 23:22:30 +0000
+++ lib/lp/buildmaster/tests/test_manager.py	2014-06-06 11:05:48 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2013 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2014 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Tests for the renovated slave scanner aka BuilddManager."""
@@ -56,11 +56,15 @@
     MockBuilder,
     OkSlave,
     TrivialBehaviour,
+    WaitingSlave,
     )
 from lp.registry.interfaces.distribution import IDistributionSet
 from lp.services.config import config
 from lp.services.log.logger import BufferLogger
 from lp.soyuz.interfaces.binarypackagebuild import IBinaryPackageBuildSet
+from lp.soyuz.model.binarypackagebuildbehaviour import (
+    BinaryPackageBuildBehaviour,
+    )
 from lp.testing import (
     ANONYMOUS,
     login,
@@ -80,7 +84,7 @@
 from lp.testing.sampledata import BOB_THE_BUILDER_NAME
 
 
-class TestSlaveScannerScan(TestCase):
+class TestSlaveScannerScan(TestCaseWithFactory):
     """Tests `SlaveScanner.scan` method.
 
     This method uses the old framework for scanning and dispatching builds.
@@ -227,7 +231,7 @@
         self.assertTrue(builder.builderok)
 
         job = getUtility(IBuildQueueSet).get(job.id)
-        self.assertBuildingJob(job, builder, logtail='This is a build log')
+        self.assertBuildingJob(job, builder, logtail='This is a build log: 0')
 
     def testScanUpdatesBuildingJobs(self):
         # Enable sampledata builder attached to an appropriate testing
@@ -530,6 +534,84 @@
         self.assertEqual(1, slave.call_log.count("resume"))
         self.assertEqual(BuildStatus.CANCELLED, build.status)
 
+    @defer.inlineCallbacks
+    def test_end_to_end(self):
+        # Test that SlaveScanner.scan() successfully finds, dispatches,
+        # collects and cleans a build.
+        build = self.factory.makeBinaryPackageBuild()
+        build.distro_arch_series.addOrUpdateChroot(
+            self.factory.makeLibraryFileAlias(db_only=True))
+        bq = build.queueBuild()
+        bq.manualScore(9000)
+
+        builder = self.factory.makeBuilder(
+            processors=[bq.processor], manual=False, vm_host='VMHOST')
+        transaction.commit()
+
+        # Mock out the build behaviour's _handleStatus_OK so it doesn't
+        # try to upload things to the librarian or queue.
+        @defer.inlineCallbacks
+        def handleStatus_OK(self, slave_status, logger, notify):
+            build.updateStatus(
+                BuildStatus.UPLOADING, builder, slave_status=slave_status)
+            transaction.commit()
+            yield self._slave.clean()
+            bq.destroySelf()
+            transaction.commit()
+        self.patch(
+            BinaryPackageBuildBehaviour, '_handleStatus_OK', handleStatus_OK)
+
+        # And create a SlaveScanner with a slave and a clock that we
+        # control.
+        get_slave = FakeMethod(OkSlave())
+        clock = task.Clock()
+        scanner = SlaveScanner(
+            builder.name, BuilderFactory(), BufferLogger(),
+            slave_factory=get_slave, clock=clock)
+
+        # The slave is idle and there's a build candidate, so the first
+        # scan will reset the builder and dispatch the build.
+        yield scanner.scan()
+        self.assertEqual(
+            ['status', 'resume', 'echo', 'ensurepresent', 'build'],
+            get_slave.result.method_log)
+        self.assertEqual(bq, builder.currentjob)
+        self.assertEqual(BuildQueueStatus.RUNNING, bq.status)
+        self.assertEqual(BuildStatus.BUILDING, build.status)
+
+        # build() has been called, so switch in a BUILDING slave.
+        # Scans will now just do a status() each, as the logtail is
+        # updated.
+        get_slave.result = BuildingSlave(
+            IBuildFarmJobBehaviour(build).getBuildCookie())
+        yield scanner.scan()
+        self.assertEqual("This is a build log: 0", bq.logtail)
+        yield scanner.scan()
+        self.assertEqual("This is a build log: 1", bq.logtail)
+        yield scanner.scan()
+        self.assertEqual("This is a build log: 2", bq.logtail)
+        self.assertEqual(
+            ['status', 'status', 'status'], get_slave.result.method_log)
+
+        # When the build finishes, the scanner will notice, call
+        # handleStatus(), and then clean the builder.  Our fake
+        # _handleStatus_OK doesn't do anything special, but there'd
+        # usually be file retrievals in the middle.
+        get_slave.result = WaitingSlave(
+            build_id=IBuildFarmJobBehaviour(build).getBuildCookie())
+        yield scanner.scan()
+        self.assertEqual(['status', 'clean'], get_slave.result.method_log)
+        self.assertIs(None, builder.currentjob)
+        self.assertEqual(BuildStatus.UPLOADING, build.status)
+        self.assertEqual(builder, build.builder)
+
+        # We're clean, so let's flip back to an idle slave and
+        # confirm that a scan does nothing special.
+        get_slave.result = OkSlave()
+        yield scanner.scan()
+        self.assertEqual(['status'], get_slave.result.method_log)
+        self.assertIs(None, builder.currentjob)
+
 
 class TestPrefetchedBuilderFactory(TestCaseWithFactory):
 

=== modified file 'lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py'
--- lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py	2014-02-03 14:43:08 +0000
+++ lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py	2014-06-06 11:05:48 +0000
@@ -1,4 +1,4 @@
-# Copyright 2010-2013 Canonical Ltd.  This software is licensed under the
+# Copyright 2010-2014 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Tests for BinaryPackageBuildBehaviour."""
@@ -403,7 +403,7 @@
         # The builder is still building the package.
         def got_update(ignored):
             # The fake log is returned from the BuildingSlave() mock.
-            self.assertEqual("This is a build log", self.candidate.logtail)
+            self.assertEqual("This is a build log: 0", self.candidate.logtail)
 
         d = self.updateBuild(self.candidate, BuildingSlave())
         return d.addCallback(got_update)