launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #25143
[Merge] ~twom/launchpad:stats-in-buildd-manager into launchpad:master
Tom Wardill has proposed merging ~twom/launchpad:stats-in-buildd-manager into launchpad:master.
Commit message:
Add statsd totals for builder stats by processor
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~twom/launchpad/+git/launchpad/+merge/389011
It'd be useful to graph the builder states over time, by architecture. Create a statsd client that is configured from settings (with a dummy one to prevent network traffic if not).
Use this in builddmanager in an extra (tunable) loop to prevent impingement on other manager work.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~twom/launchpad:stats-in-buildd-manager into launchpad:master.
diff --git a/configs/development/launchpad-lazr.conf b/configs/development/launchpad-lazr.conf
index b8f2c55..4cc4106 100644
--- a/configs/development/launchpad-lazr.conf
+++ b/configs/development/launchpad-lazr.conf
@@ -233,3 +233,8 @@ hostname: feeds.launchpad.test
# so disable that here. note that the testrunner config inherits
# this setting from us.
send_email: false
+
+[statsd]
+host:
+port:
+prefix:
diff --git a/constraints.txt b/constraints.txt
index 706f7da..adf37e2 100644
--- a/constraints.txt
+++ b/constraints.txt
@@ -298,6 +298,7 @@ SimpleTAL==4.3; python_version < "3"
SimpleTAL==5.2; python_version >= "3"
soupmatchers==0.4
soupsieve==1.9
+statsd==3.3.0
# lp:~launchpad-committers/storm/lp
storm==0.24+lp416
subprocess32==3.2.6
diff --git a/lib/lp/buildmaster/manager.py b/lib/lp/buildmaster/manager.py
index a4b96e3..418fe6a 100644
--- a/lib/lp/buildmaster/manager.py
+++ b/lib/lp/buildmaster/manager.py
@@ -61,6 +61,7 @@ from lp.services.database.stormexpr import (
Values,
)
from lp.services.propertycache import get_property_cache
+from lp.services.statsd import getStatsdClient
BUILDD_MANAGER_LOG_NAME = "slave-scanner"
@@ -691,6 +692,9 @@ class BuilddManager(service.Service):
# How often to flush logtail updates, in seconds.
FLUSH_LOGTAILS_INTERVAL = 15
+ # How often to update stats, in seconds
+ UPDATE_STATS_INTERVAL = 60
+
def __init__(self, clock=None, builder_factory=None):
# Use the clock if provided, it's so that tests can
# advance it. Use the reactor by default.
@@ -702,6 +706,7 @@ class BuilddManager(service.Service):
self.logger = self._setupLogger()
self.current_builders = []
self.pending_logtails = {}
+ self.statsd_client = getStatsdClient()
def _setupLogger(self):
"""Set up a 'slave-scanner' logger that redirects to twisted.
@@ -721,6 +726,27 @@ class BuilddManager(service.Service):
logger.setLevel(level)
return logger
+ def updateStats(self):
+ """Update statsd with the builder statuses."""
+ counts_by_processor = {}
+ for builder in self.builder_factory.iterVitals():
+ for processor_name in builder.processor_names:
+ counts = counts_by_processor.get(
+ processor_name,
+ {'cleaning': 0, 'ok': 0, 'disabled': 0})
+ if builder.clean_status == BuilderCleanStatus.CLEANING:
+ counts['cleaning'] += 1
+ elif builder.builderok:
+ counts['ok'] += 1
+ else:
+ counts['disabled'] += 1
+ counts_by_processor[processor_name] = counts
+ for processor, counts in counts_by_processor.items():
+ for count_name, count_value in counts.items():
+ gauge_name = "builders.{}.{}".format(processor, count_name)
+ self.logger.debug("{}: {}".format(gauge_name, count_name))
+ self.statsd_client.gauge(gauge_name, count_value)
+
def checkForNewBuilders(self):
"""Add and return any new builders."""
new_builders = set(
@@ -790,6 +816,9 @@ class BuilddManager(service.Service):
# Schedule bulk flushes for build queue logtail updates.
self.flush_logtails_loop, self.flush_logtails_deferred = (
self._startLoop(self.FLUSH_LOGTAILS_INTERVAL, self.flushLogTails))
+ # Schedule stats updates.
+ self.stats_update_loop, self.stats_update_deferred = (
+ self._startLoop(self.UPDATE_STATS_INTERVAL, self.updateStats))
def stopService(self):
"""Callback for when we need to shut down."""
@@ -798,9 +827,11 @@ class BuilddManager(service.Service):
deferreds = [slave.stopping_deferred for slave in self.builder_slaves]
deferreds.append(self.scan_builders_deferred)
deferreds.append(self.flush_logtails_deferred)
+ deferreds.append(self.stats_update_deferred)
self.flush_logtails_loop.stop()
self.scan_builders_loop.stop()
+ self.stats_update_loop.stop()
for slave in self.builder_slaves:
slave.stopCycle()
diff --git a/lib/lp/buildmaster/tests/test_manager.py b/lib/lp/buildmaster/tests/test_manager.py
index b0d56ed..01acedf 100644
--- a/lib/lp/buildmaster/tests/test_manager.py
+++ b/lib/lp/buildmaster/tests/test_manager.py
@@ -13,8 +13,12 @@ import os
import signal
import time
+from fixtures import MockPatch
from six.moves import xmlrpc_client
-from testtools.matchers import Equals
+from testtools.matchers import (
+ Equals,
+ MatchesListwise,
+ )
from testtools.testcase import ExpectedException
from testtools.twistedsupport import AsynchronousDeferredRunTest
import transaction
@@ -45,6 +49,7 @@ from lp.buildmaster.interfaces.builder import (
IBuilderSet,
)
from lp.buildmaster.interfaces.buildqueue import IBuildQueueSet
+from lp.buildmaster.interfaces.processor import IProcessorSet
from lp.buildmaster.manager import (
BuilddManager,
BUILDER_FAILURE_THRESHOLD,
@@ -1596,3 +1601,69 @@ class TestBuilddManagerScript(TestCaseWithFactory):
self.assertFalse(
os.access(rotated_logfilepath, os.F_OK),
"Twistd's log file was rotated by twistd.")
+
+
+class TestStats(TestCaseWithFactory):
+
+ layer = ZopelessDatabaseLayer
+ run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=20)
+
+ def setUp(self):
+ super(TestStats, self).setUp()
+ self.stats_client = self.useFixture(
+ MockPatch(
+ 'lp.buildmaster.manager.getStatsdClient'
+ )).mock()
+
+ def test_single_processor(self):
+ builder = self.factory.makeBuilder()
+ builder.setCleanStatus(BuilderCleanStatus.CLEAN)
+ self.patch(BuilderSlave, 'makeBuilderSlave', FakeMethod(OkSlave()))
+ transaction.commit()
+ clock = task.Clock()
+ manager = BuilddManager(clock=clock)
+ manager.builder_factory.update()
+ manager.updateStats()
+
+ self.assertEqual(3, self.stats_client.gauge.call_count)
+ for call in self.stats_client.mock.gauge.call_args_list:
+ self.assertIn('386', call[0][0])
+
+ def test_multiple_processor(self):
+ builder = self.factory.makeBuilder(
+ processors=[getUtility(IProcessorSet).getByName('amd64')])
+ builder.setCleanStatus(BuilderCleanStatus.CLEAN)
+ self.patch(BuilderSlave, 'makeBuilderSlave', FakeMethod(OkSlave()))
+ transaction.commit()
+ clock = task.Clock()
+ manager = BuilddManager(clock=clock)
+ manager.builder_factory.update()
+ manager.updateStats()
+
+ self.assertEqual(6, self.stats_client.gauge.call_count)
+ i386_calls = [c for c in self.stats_client.gauge.call_args_list
+ if '386' in c[0][0]]
+ amd64_calls = [c for c in self.stats_client.gauge.call_args_list
+ if 'amd64' in c[0][0]]
+ self.assertEqual(3, len(i386_calls))
+ self.assertEqual(3, len(amd64_calls))
+
+ def test_correct_values(self):
+ builder = self.factory.makeBuilder(
+ processors=[getUtility(IProcessorSet).getByName('amd64')])
+ builder.setCleanStatus(BuilderCleanStatus.CLEANING)
+ self.patch(BuilderSlave, 'makeBuilderSlave', FakeMethod(OkSlave()))
+ transaction.commit()
+ clock = task.Clock()
+ manager = BuilddManager(clock=clock)
+ manager.builder_factory.update()
+ manager.updateStats()
+
+ self.assertEqual(6, self.stats_client.gauge.call_count)
+ calls = [c[0] for c in self.stats_client.gauge.call_args_list
+ if 'amd64' in c[0][0]]
+ self.assertThat(
+ calls, MatchesListwise(
+ [Equals(('builders.amd64.disabled', 0)),
+ Equals(('builders.amd64.ok', 0)),
+ Equals(('builders.amd64.cleaning', 1))]))
diff --git a/lib/lp/services/config/schema-lazr.conf b/lib/lp/services/config/schema-lazr.conf
index 80bc7e2..bb8afcd 100644
--- a/lib/lp/services/config/schema-lazr.conf
+++ b/lib/lp/services/config/schema-lazr.conf
@@ -113,7 +113,6 @@ authentication_endpoint: none
# authserver.
authentication_timeout: 15
-
[canonical]
# datatype: boolean
show_tracebacks: False
@@ -2022,3 +2021,9 @@ concurrency: 1
timeout: 86400
fallback_queue:
concurrency: 1
+
+# Statsd connection details
+[statsd]
+host: none
+port: none
+prefix: none
diff --git a/lib/lp/services/statsd/__init__.py b/lib/lp/services/statsd/__init__.py
new file mode 100644
index 0000000..41acbf2
--- /dev/null
+++ b/lib/lp/services/statsd/__init__.py
@@ -0,0 +1,35 @@
+# Copyright 2020 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Statsd client wrapper with Launchpad configuration"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = ['getStatsdClient']
+
+
+from statsd import StatsClient
+
+from lp.services.config import config
+
+
+class UnconfiguredStatsdClient:
+ """Dummy client for if statsd is not configured in the environment.
+
+ This client will be used if the statsd settings are not available to
+ Launchpad. Prevents unnecessary network traffic.
+ """
+
+ def __getattr__(self, name):
+ return lambda *args, **kwargs: None
+
+
+def getStatsdClient():
+ if getattr(config, 'statsd', None) and config.statsd.host:
+ return StatsClient(
+ host=config.statsd.host,
+ port=config.statsd.port,
+ prefix=config.statsd.prefix)
+ else:
+ return UnconfiguredStatsdClient()
diff --git a/lib/lp/services/statsd/tests/__init__.py b/lib/lp/services/statsd/tests/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/lib/lp/services/statsd/tests/__init__.py
diff --git a/lib/lp/services/statsd/tests/test_lp_statsd.py b/lib/lp/services/statsd/tests/test_lp_statsd.py
new file mode 100644
index 0000000..6a244aa
--- /dev/null
+++ b/lib/lp/services/statsd/tests/test_lp_statsd.py
@@ -0,0 +1,40 @@
+# Copyright 2020 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Tests for the Launchpad Statsd Client"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+
+from statsd import StatsClient
+
+from lp.services.config import config
+from lp.services.statsd import (
+ getStatsdClient,
+ UnconfiguredStatsdClient,
+ )
+from lp.testing import TestCase
+from lp.testing.layers import BaseLayer
+
+
+class TestClientConfiguration(TestCase):
+
+ layer = BaseLayer
+
+ def test_get_correct_instance_unconfigured(self):
+ """Test that we get the correct client, depending on config values."""
+ config.push(
+ 'statsd_test',
+ "[statsd]\nhost:")
+ client = getStatsdClient()
+ self.assertEqual(
+ type(client), UnconfiguredStatsdClient)
+
+ def test_get_correct_instance_configured(self):
+ config.push(
+ 'statsd_test',
+ "[statsd]\nhost: 127.0.01\nport: 9999\nprefix: test\n")
+ client = getStatsdClient()
+ self.assertEqual(
+ type(client), StatsClient)
diff --git a/setup.py b/setup.py
index b724650..49e5306 100644
--- a/setup.py
+++ b/setup.py
@@ -232,6 +232,7 @@ setup(
'six',
'soupmatchers',
'Sphinx',
+ 'statsd',
'storm',
# XXX cjwatson 2020-08-07: Temporarily dropped on Python 3 until
# codeimport can be ported to Breezy.
