launchpad-reviewers team mailing list archive

[Tom Wardill <tom.wardill@xxxxxxxxxxxxx>]

Tom Wardill has proposed merging ~twom/launchpad:stats-actual-build-queues into launchpad:master with ~twom/launchpad:stats-queues-in-buildd as a prerequisite.

Commit message:
Add buildqueue sizes

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~twom/launchpad/+git/launchpad/+merge/389729

Use getBuildQueueSizes to get a simple report on the state of the build queues by architecture and virtualisation state.

Relies on the stats client refactoring in lp:~twom/launchpad/stats-queues-in-buildd
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~twom/launchpad:stats-actual-build-queues into launchpad:master.

diff --git a/lib/lp/buildmaster/manager.py b/lib/lp/buildmaster/manager.py
index 5e8e88a..403a3ba 100644
--- a/lib/lp/buildmaster/manager.py
+++ b/lib/lp/buildmaster/manager.py
@@ -722,7 +722,7 @@ class BuilddManager(service.Service):
 
         Make it less verbose to avoid messing too much with the old code.
         """
-        level = logging.INFO
+        level = logging.DEBUG
         logger = logging.getLogger(BUILDD_MANAGER_LOG_NAME)
         logger.propagate = False
 
@@ -735,7 +735,7 @@ class BuilddManager(service.Service):
         logger.setLevel(level)
         return logger
 
-    def updateStats(self):
+    def _updateBuilderCounts(self):
         """Update statsd with the builder statuses."""
         self.logger.debug("Updating builder stats.")
         counts_by_processor = {}
@@ -765,6 +765,22 @@ class BuilddManager(service.Service):
                 self.statsd_client.gauge(gauge_name, count_value)
         self.logger.debug("Builder stats update complete.")
 
+    def _updateBuilderQueues(self):
+        """Update statsd with the build queue lengths."""
+        self.logger.debug("Updating build queue stats.")
+        queue_details = getUtility(IBuilderSet).getBuildQueueSizes()
+        for queue_type, contents in queue_details.items():
+            virt = True if queue_type == 'virt' else False
+            for arch, value in contents.items():
+                gauge_name = "buildqueue,virtualized={},arch={}".format(
+                    virt, arch)
+                self.statsd_client.gauge(gauge_name, value[0])
+        self.logger.debug("Build queue stats update complete.")
+
+    def updateStats(self):
+        self._updateBuilderCounts()
+        self._updateBuilderQueues()
+
     def checkForNewBuilders(self):
         """Add and return any new builders."""
         new_builders = set(
diff --git a/lib/lp/buildmaster/tests/test_manager.py b/lib/lp/buildmaster/tests/test_manager.py
index 8d23139..5f5cc55 100644
--- a/lib/lp/buildmaster/tests/test_manager.py
+++ b/lib/lp/buildmaster/tests/test_manager.py
@@ -1658,13 +1658,14 @@ class TestStats(StatsMixin, TestCaseWithFactory):
         super(TestStats, self).setUp()
         self.setUpStats()
 
-    def test_single_processor(self):
+    def test_single_processor_counts(self):
         builder = self.factory.makeBuilder()
         builder.setCleanStatus(BuilderCleanStatus.CLEAN)
         self.patch(BuilderSlave, 'makeBuilderSlave', FakeMethod(OkSlave()))
         transaction.commit()
         clock = task.Clock()
         manager = BuilddManager(clock=clock)
+        manager._updateBuilderQueues = FakeMethod()
         manager.builder_factory.update()
         manager.updateStats()
 
@@ -1672,7 +1673,7 @@ class TestStats(StatsMixin, TestCaseWithFactory):
         for call in self.stats_client.mock.gauge.call_args_list:
             self.assertIn('386', call[0][0])
 
-    def test_multiple_processor(self):
+    def test_multiple_processor_counts(self):
         builder = self.factory.makeBuilder(
             processors=[getUtility(IProcessorSet).getByName('amd64')])
         builder.setCleanStatus(BuilderCleanStatus.CLEAN)
@@ -1680,6 +1681,7 @@ class TestStats(StatsMixin, TestCaseWithFactory):
         transaction.commit()
         clock = task.Clock()
         manager = BuilddManager(clock=clock)
+        manager._updateBuilderQueues = FakeMethod()
         manager.builder_factory.update()
         manager.updateStats()
 
@@ -1691,7 +1693,7 @@ class TestStats(StatsMixin, TestCaseWithFactory):
         self.assertEqual(8, len(i386_calls))
         self.assertEqual(4, len(amd64_calls))
 
-    def test_correct_values(self):
+    def test_correct_values_counts(self):
         builder = self.factory.makeBuilder(
             processors=[getUtility(IProcessorSet).getByName('amd64')])
         builder.setCleanStatus(BuilderCleanStatus.CLEANING)
@@ -1699,6 +1701,7 @@ class TestStats(StatsMixin, TestCaseWithFactory):
         transaction.commit()
         clock = task.Clock()
         manager = BuilddManager(clock=clock)
+        manager._updateBuilderQueues = FakeMethod()
         manager.builder_factory.update()
         manager.updateStats()
 
@@ -1712,3 +1715,26 @@ class TestStats(StatsMixin, TestCaseWithFactory):
                  Equals(('builders.idle,arch=amd64,virtualized=True', 0)),
                  Equals(('builders.cleaning,arch=amd64,virtualized=True', 1))
                  ]))
+
+    def test_updateBuilderQueues(self):
+        builder = self.factory.makeBuilder(
+            processors=[getUtility(IProcessorSet).getByName('amd64')])
+        builder.setCleanStatus(BuilderCleanStatus.CLEANING)
+        build = self.factory.makeSnapBuild()
+        build.queueBuild()
+        self.patch(BuilderSlave, 'makeBuilderSlave', FakeMethod(OkSlave()))
+        transaction.commit()
+        clock = task.Clock()
+        manager = BuilddManager(clock=clock)
+        manager._updateBuilderCounts = FakeMethod()
+        manager.builder_factory.update()
+        manager.updateStats()
+
+        self.assertEqual(2, self.stats_client.gauge.call_count)
+        self.assertThat(
+            [x[0] for x in self.stats_client.gauge.call_args_list],
+            MatchesListwise(
+                [Equals(('buildqueue,virtualized=True,arch={}'.format(
+                    build.processor.name), 1)),
+                 Equals(('buildqueue,virtualized=False,arch=386', 1))
+                 ]))