← Back to team overview

launchpad-reviewers team mailing list archive

[Merge] ~cjwatson/launchpad:statsd-labels-dict into launchpad:master

 

Colin Watson has proposed merging ~cjwatson/launchpad:statsd-labels-dict into launchpad:master.

Commit message:
Pass statsd metric labels as a dict

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/399228

This is more convenient than having to deal with the line protocol framing at every call site.
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:statsd-labels-dict into launchpad:master.
diff --git a/lib/lp/buildmaster/manager.py b/lib/lp/buildmaster/manager.py
index 1819b29..9e58a34 100644
--- a/lib/lp/buildmaster/manager.py
+++ b/lib/lp/buildmaster/manager.py
@@ -529,13 +529,16 @@ class SlaveScanner:
         builder = self.builder_factory[self.builder_name]
         try:
             builder.gotFailure()
+            labels = {}
             if builder.current_build is not None:
                 builder.current_build.gotFailure()
-                self.statsd_client.incr(
-                    'builders.judged_failed,build=True,arch={}'.format(
-                        builder.current_build.processor.name))
+                labels.update({
+                    'build': True,
+                    'arch': builder.current_build.processor.name,
+                    })
             else:
-                self.statsd_client.incr('builders.judged_failed,build=False')
+                labels['build'] = False
+            self.statsd_client.incr('builders.judged_failed', labels=labels)
             recover_failure(self.logger, vitals, builder, retry, failure.value)
             transaction.commit()
         except Exception:
diff --git a/lib/lp/buildmaster/model/buildfarmjobbehaviour.py b/lib/lp/buildmaster/model/buildfarmjobbehaviour.py
index 8fab8ae..f8b5034 100644
--- a/lib/lp/buildmaster/model/buildfarmjobbehaviour.py
+++ b/lib/lp/buildmaster/model/buildfarmjobbehaviour.py
@@ -159,9 +159,11 @@ class BuildFarmJobBehaviourBase:
         job_type_name = job_type.name if job_type else 'UNKNOWN'
         statsd_client = getUtility(IStatsdClient)
         statsd_client.incr(
-            'build.count,job_type={},builder_name={}'.format(
-                job_type_name,
-                self._builder.name))
+            'build.count',
+            labels={
+                'job_type': job_type_name,
+                'builder_name': self._builder.name,
+                })
 
         logger.info(
             "Job %s (%s) started on %s: %s %s"
diff --git a/lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py b/lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py
index dd3a313..dc548c4 100644
--- a/lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py
+++ b/lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py
@@ -278,8 +278,8 @@ class TestDispatchBuildToSlave(StatsMixin, TestCase):
         self.assertEqual(1, self.stats_client.incr.call_count)
         self.assertEqual(
             self.stats_client.incr.call_args_list[0][0],
-            ('build.count,job_type=UNKNOWN,'
-             'builder_name=mock-builder,env=test',))
+            ('build.count,builder_name=mock-builder,env=test,'
+             'job_type=UNKNOWN',))
 
 
 class TestGetUploadMethodsMixin:
diff --git a/lib/lp/oci/tests/test_ocirecipebuildbehaviour.py b/lib/lp/oci/tests/test_ocirecipebuildbehaviour.py
index 29a18e1..16012f5 100644
--- a/lib/lp/oci/tests/test_ocirecipebuildbehaviour.py
+++ b/lib/lp/oci/tests/test_ocirecipebuildbehaviour.py
@@ -576,8 +576,8 @@ class TestAsyncOCIRecipeBuildBehaviour(
         self.assertEqual(1, self.stats_client.incr.call_count)
         self.assertEqual(
             self.stats_client.incr.call_args_list[0][0],
-            ('build.count,job_type=OCIRECIPEBUILD,'
-             'builder_name={},env=test'.format(
+            ('build.count,builder_name={},env=test,'
+             'job_type=OCIRECIPEBUILD'.format(
                 builder.name),))
 
     @defer.inlineCallbacks
diff --git a/lib/lp/oci/tests/test_ocirecipebuildjob.py b/lib/lp/oci/tests/test_ocirecipebuildjob.py
index 59554e6..8afbc08 100644
--- a/lib/lp/oci/tests/test_ocirecipebuildjob.py
+++ b/lib/lp/oci/tests/test_ocirecipebuildjob.py
@@ -297,11 +297,11 @@ class TestOCIRegistryUploadJob(TestCaseWithFactory, MultiArchRecipeMixin,
         self.assertEqual(4, self.stats_client.incr.call_count)
         calls = [x[0][0] for x in self.stats_client.incr.call_args_list]
         self.assertThat(calls, MatchesListwise([
-            Equals('job.start_count,type=OCIRecipeRequestBuildsJob,env=test'),
+            Equals('job.start_count,env=test,type=OCIRecipeRequestBuildsJob'),
             Equals(
-                'job.complete_count,type=OCIRecipeRequestBuildsJob,env=test'),
-            Equals('job.start_count,type=OCIRegistryUploadJob,env=test'),
-            Equals('job.complete_count,type=OCIRegistryUploadJob,env=test')]))
+                'job.complete_count,env=test,type=OCIRecipeRequestBuildsJob'),
+            Equals('job.start_count,env=test,type=OCIRegistryUploadJob'),
+            Equals('job.complete_count,env=test,type=OCIRegistryUploadJob')]))
 
     def test_run_multiple_architectures(self):
         build_request = self.makeBuildRequest()
@@ -336,13 +336,13 @@ class TestOCIRegistryUploadJob(TestCaseWithFactory, MultiArchRecipeMixin,
             client.uploadManifestList.calls)
         calls = [x[0][0] for x in self.stats_client.incr.call_args_list]
         self.assertThat(calls, MatchesListwise([
-            Equals('job.start_count,type=OCIRecipeRequestBuildsJob,env=test'),
+            Equals('job.start_count,env=test,type=OCIRecipeRequestBuildsJob'),
             Equals(
-                'job.complete_count,type=OCIRecipeRequestBuildsJob,env=test'),
-            Equals('job.start_count,type=OCIRegistryUploadJob,env=test'),
-            Equals('job.complete_count,type=OCIRegistryUploadJob,env=test'),
-            Equals('job.start_count,type=OCIRegistryUploadJob,env=test'),
-            Equals('job.complete_count,type=OCIRegistryUploadJob,env=test')]))
+                'job.complete_count,env=test,type=OCIRecipeRequestBuildsJob'),
+            Equals('job.start_count,env=test,type=OCIRegistryUploadJob'),
+            Equals('job.complete_count,env=test,type=OCIRegistryUploadJob'),
+            Equals('job.start_count,env=test,type=OCIRegistryUploadJob'),
+            Equals('job.complete_count,env=test,type=OCIRegistryUploadJob')]))
 
     def test_failing_upload_does_not_retries_automatically(self):
         build_request = self.makeBuildRequest(include_i386=False)
diff --git a/lib/lp/services/job/runner.py b/lib/lp/services/job/runner.py
index 5368050..1e8b0e6 100644
--- a/lib/lp/services/job/runner.py
+++ b/lib/lp/services/job/runner.py
@@ -329,20 +329,21 @@ class BaseRunnableJob(BaseRunnableJobSource):
         """See `IJob`."""
         self.job.start(manage_transaction=manage_transaction)
         statsd = getUtility(IStatsdClient)
-        statsd.incr('job.start_count,type={}'.format(self.__class__.__name__))
+        statsd.incr(
+            'job.start_count', labels={'type': self.__class__.__name__})
 
     def complete(self, manage_transaction=False):
         """See `IJob`."""
         self.job.complete(manage_transaction=manage_transaction)
         statsd = getUtility(IStatsdClient)
-        statsd.incr('job.complete_count,type={}'.format(
-            self.__class__.__name__))
+        statsd.incr(
+            'job.complete_count', labels={'type': self.__class__.__name__})
 
     def fail(self, manage_transaction=False):
         """See `IJob`."""
         self.job.fail(manage_transaction=manage_transaction)
         statsd = getUtility(IStatsdClient)
-        statsd.incr('job.fail_count,type={}'.format(self.__class__.__name__))
+        statsd.incr('job.fail_count', labels={'type': self.__class__.__name__})
 
 
 class BaseJobRunner(LazrJobRunner):
diff --git a/lib/lp/services/job/tests/test_runner.py b/lib/lp/services/job/tests/test_runner.py
index 36842b9..d14194c 100644
--- a/lib/lp/services/job/tests/test_runner.py
+++ b/lib/lp/services/job/tests/test_runner.py
@@ -181,10 +181,10 @@ class TestJobRunner(StatsMixin, TestCaseWithFactory):
         self.assertEqual([job_1], runner.completed_jobs)
         self.assertEqual(
             self.stats_client.incr.call_args_list[0][0],
-            ('job.start_count,type=NullJob,env=test',))
+            ('job.start_count,env=test,type=NullJob',))
         self.assertEqual(
             self.stats_client.incr.call_args_list[1][0],
-            ('job.complete_count,type=NullJob,env=test',))
+            ('job.complete_count,env=test,type=NullJob',))
 
     def test_runAll(self):
         """Ensure runAll works in the normal case."""
@@ -232,10 +232,10 @@ class TestJobRunner(StatsMixin, TestCaseWithFactory):
         self.assertEqual(["{'foo': 'bar'}"], list(oops['req_vars'].values()))
         self.assertEqual(
             self.stats_client.incr.call_args_list[0][0],
-            ('job.start_count,type=NullJob,env=test',))
+            ('job.start_count,env=test,type=NullJob',))
         self.assertEqual(
             self.stats_client.incr.call_args_list[1][0],
-            ('job.fail_count,type=NullJob,env=test',))
+            ('job.fail_count,env=test,type=NullJob',))
 
     def test_oops_messages_used_when_handling(self):
         """Oops messages should appear even when exceptions are handled."""
diff --git a/lib/lp/services/scripts/doc/script-monitoring.txt b/lib/lp/services/scripts/doc/script-monitoring.txt
index df81ad5..89a6406 100644
--- a/lib/lp/services/scripts/doc/script-monitoring.txt
+++ b/lib/lp/services/scripts/doc/script-monitoring.txt
@@ -72,7 +72,7 @@ It sends a corresponding timing stat to statsd.
     >>> stats_client.timing.call_count
     1
     >>> print(stats_client.timing.call_args[0][0])
-    script_activity,name=script-name,env=test
+    script_activity,env=test,name=script-name
     >>> stats_client.timing.call_args[0][1]
     60000.0
 
diff --git a/lib/lp/services/scripts/model/scriptactivity.py b/lib/lp/services/scripts/model/scriptactivity.py
index 0df3849..6b11bbd 100644
--- a/lib/lp/services/scripts/model/scriptactivity.py
+++ b/lib/lp/services/scripts/model/scriptactivity.py
@@ -63,8 +63,9 @@ class ScriptActivitySet:
         # Pass equivalent information through to statsd as well.  (Don't
         # bother with the hostname, since telegraf adds that.)
         getUtility(IStatsdClient).timing(
-            'script_activity,name={}'.format(name),
-            (date_completed - date_started).total_seconds() * 1000)
+            'script_activity',
+            (date_completed - date_started).total_seconds() * 1000,
+            labels={'name': name})
         return activity
 
     def getLastActivity(self, name):
diff --git a/lib/lp/services/statsd/interfaces/statsd_client.py b/lib/lp/services/statsd/interfaces/statsd_client.py
index fa10401..1d8d4a9 100644
--- a/lib/lp/services/statsd/interfaces/statsd_client.py
+++ b/lib/lp/services/statsd/interfaces/statsd_client.py
@@ -22,3 +22,9 @@ class IStatsdClient(Interface):
 
     def reload():
         """Reload the statsd client configuration."""
+
+    def composeMetric(name, labels):
+        """Compose a full metric name from a measurement name and labels.
+
+        The inputs are composed according to the InfluxDB line protocol.
+        """
diff --git a/lib/lp/services/statsd/model/statsd_client.py b/lib/lp/services/statsd/model/statsd_client.py
index 7d6c3c1..6b924d2 100644
--- a/lib/lp/services/statsd/model/statsd_client.py
+++ b/lib/lp/services/statsd/model/statsd_client.py
@@ -8,6 +8,7 @@ from __future__ import absolute_import, print_function, unicode_literals
 __metaclass__ = type
 __all__ = ['StatsdClient']
 
+import re
 
 from statsd import StatsClient
 from zope.interface import implementer
@@ -40,16 +41,40 @@ class StatsdClient:
             self._client = None
 
     def reload(self):
+        """See `IStatsdClient`."""
         self._make_client()
 
+    def _escapeMeasurement(self, measurement):
+        # Escape a measurement name for the InfluxDB line protocol:
+        #   https://docs.influxdata.com/influxdb/cloud/reference/syntax/\
+        #       line-protocol/
+        return re.sub(r"([, \\])", r"\\\1", measurement)
+
+    def _escapeTag(self, tag):
+        # Escape a tag key or value for the InfluxDB line protocol:
+        #   https://docs.influxdata.com/influxdb/cloud/reference/syntax/\
+        #       line-protocol/
+        return re.sub(r"([,= \\])", r"\\\1", tag)
+
+    def composeMetric(self, name, labels):
+        """See `IStatsdClient`."""
+        if labels is None:
+            labels = {}
+        elements = [self._escapeMeasurement(name)]
+        for key, value in sorted(labels.items()):
+            elements.append("{}={}".format(
+                self._escapeTag(key), self._escapeTag(str(value))))
+        return ",".join(elements)
+
     def __getattr__(self, name):
         if self._client is not None:
             wrapped = getattr(self._client, name)
             if name in ("timer", "timing", "incr", "decr", "gauge", "set"):
                 def wrapper(stat, *args, **kwargs):
+                    labels = kwargs.pop("labels", None) or {}
+                    labels["env"] = config.statsd.environment
                     return wrapped(
-                        "%s,env=%s" % (stat, config.statsd.environment),
-                        *args, **kwargs)
+                        self.composeMetric(stat, labels), *args, **kwargs)
 
                 return wrapper
             else:
diff --git a/lib/lp/services/statsd/numbercruncher.py b/lib/lp/services/statsd/numbercruncher.py
index a90a815..6add2e4 100644
--- a/lib/lp/services/statsd/numbercruncher.py
+++ b/lib/lp/services/statsd/numbercruncher.py
@@ -80,9 +80,10 @@ class NumberCruncher(service.Service):
         stopping_deferred = loop.start(interval)
         return loop, stopping_deferred
 
-    def _sendGauge(self, gauge_name, value):
-        self.logger.debug("{}: {}".format(gauge_name, value))
-        self.statsd_client.gauge(gauge_name, value)
+    def _sendGauge(self, gauge_name, value, labels=None):
+        self.logger.debug("{}: {}".format(
+            self.statsd_client.composeMetric(gauge_name, labels), value))
+        self.statsd_client.gauge(gauge_name, value, labels=labels)
 
     def updateBuilderQueues(self):
         """Update statsd with the build queue lengths.
@@ -95,9 +96,9 @@ class NumberCruncher(service.Service):
             for queue_type, contents in queue_details.items():
                 virt = queue_type == 'virt'
                 for arch, value in contents.items():
-                    gauge_name = (
-                        "buildqueue,virtualized={},arch={}".format(virt, arch))
-                    self._sendGauge(gauge_name, value[0])
+                    self._sendGauge(
+                        "buildqueue", value[0],
+                        labels={"virtualized": virt, "arch": arch})
             self.logger.debug("Build queue stats update complete.")
         except Exception:
             self.logger.exception("Failure while updating build queue stats:")
@@ -115,9 +116,7 @@ class NumberCruncher(service.Service):
                 continue
             for processor_name in builder.processor_names:
                 counts = counts_by_processor.setdefault(
-                    "{},virtualized={}".format(
-                        processor_name,
-                        builder.virtualized),
+                    (processor_name, builder.virtualized),
                     {'cleaning': 0, 'idle': 0, 'disabled': 0, 'building': 0})
                 if not builder.builderok:
                     counts['disabled'] += 1
@@ -127,11 +126,15 @@ class NumberCruncher(service.Service):
                     counts['building'] += 1
                 elif builder.clean_status == BuilderCleanStatus.CLEAN:
                     counts['idle'] += 1
-        for processor, counts in counts_by_processor.items():
+        for (processor, virtualized), counts in counts_by_processor.items():
             for count_name, count_value in counts.items():
-                gauge_name = "builders,status={},arch={}".format(
-                    count_name, processor)
-                self._sendGauge(gauge_name, count_value)
+                self._sendGauge(
+                    "builders", count_value,
+                    labels={
+                        "status": count_name,
+                        "arch": processor,
+                        "virtualized": virtualized,
+                        })
         self.logger.debug("Builder stats update complete.")
 
     def updateBuilderStats(self):
diff --git a/lib/lp/services/statsd/tests/test_numbercruncher.py b/lib/lp/services/statsd/tests/test_numbercruncher.py
index 37f8bd3..a22938b 100644
--- a/lib/lp/services/statsd/tests/test_numbercruncher.py
+++ b/lib/lp/services/statsd/tests/test_numbercruncher.py
@@ -131,17 +131,17 @@ class TestNumberCruncher(StatsMixin, TestCaseWithFactory):
         self.assertThat(
             calls, MatchesSetwise(
                 Equals((
-                    'builders,status=disabled,arch=amd64,'
-                    'virtualized=True,env=test', 0)),
+                    'builders,arch=amd64,env=test,status=disabled,'
+                    'virtualized=True', 0)),
                 Equals((
-                    'builders,status=building,arch=amd64,'
-                    'virtualized=True,env=test', 2)),
+                    'builders,arch=amd64,env=test,status=building,'
+                    'virtualized=True', 2)),
                 Equals((
-                    'builders,status=idle,arch=amd64,'
-                    'virtualized=True,env=test', 4)),
+                    'builders,arch=amd64,env=test,status=idle,'
+                    'virtualized=True', 4)),
                 Equals((
-                    'builders,status=cleaning,arch=amd64,'
-                    'virtualized=True,env=test', 3))
+                    'builders,arch=amd64,env=test,status=cleaning,'
+                    'virtualized=True', 3))
                 ))
 
     def test_updateBuilderStats_error(self):
@@ -174,9 +174,9 @@ class TestNumberCruncher(StatsMixin, TestCaseWithFactory):
         self.assertThat(
             [x[0] for x in self.stats_client.gauge.call_args_list],
             MatchesSetwise(
-                Equals(('buildqueue,virtualized=True,arch={},env=test'.format(
+                Equals(('buildqueue,arch={},env=test,virtualized=True'.format(
                     build.processor.name), 1)),
-                Equals(('buildqueue,virtualized=False,arch=386,env=test', 1))
+                Equals(('buildqueue,arch=386,env=test,virtualized=False', 1))
                 ))
 
     def test_updateBuilderQueues_error(self):
diff --git a/lib/lp/services/webapp/publication.py b/lib/lp/services/webapp/publication.py
index c75ff69..37bf494 100644
--- a/lib/lp/services/webapp/publication.py
+++ b/lib/lp/services/webapp/publication.py
@@ -495,10 +495,12 @@ class LaunchpadBrowserPublication(
                 publication_thread_duration)
         # Update statsd, timing is in milliseconds
         getUtility(IStatsdClient).timing(
-            'publication_duration,success=True,pageid={}'.format(
-                self._prepPageIDForMetrics(
-                    request._orig_env.get('launchpad.pageid'))),
-            publication_duration * 1000)
+            'publication_duration', publication_duration * 1000,
+            labels={
+                'success': True,
+                'pageid': self._prepPageIDForMetrics(
+                    request._orig_env.get('launchpad.pageid')),
+                })
 
         # Calculate SQL statement statistics.
         sql_statements = da.get_request_statements()
@@ -611,9 +613,11 @@ class LaunchpadBrowserPublication(
                 'launchpad.traversalthreadduration', traversal_thread_duration)
         # Update statsd, timing is in milliseconds
         getUtility(IStatsdClient).timing(
-            'traversal_duration,success=True,pageid={}'.format(
-                self._prepPageIDForMetrics(pageid)),
-            traversal_duration * 1000)
+            'traversal_duration', traversal_duration * 1000,
+            labels={
+                'success': True,
+                'pageid': self._prepPageIDForMetrics(pageid),
+                })
 
     def _maybePlacefullyAuthenticate(self, request, ob):
         """ This should never be called because we've excised it in
@@ -649,10 +653,12 @@ class LaunchpadBrowserPublication(
                     publication_thread_duration)
             # Update statsd, timing is in milliseconds
             getUtility(IStatsdClient).timing(
-                'publication_duration,success=False,pageid={}'.format(
-                    self._prepPageIDForMetrics(
-                        request._orig_env.get('launchpad.pageid'))),
-                publication_duration * 1000)
+                'publication_duration', publication_duration * 1000,
+                labels={
+                    'success': False,
+                    'pageid': self._prepPageIDForMetrics(
+                        request._orig_env.get('launchpad.pageid')),
+                    })
         elif (hasattr(request, '_traversal_start') and
               ('launchpad.traversalduration' not in orig_env)):
             # The traversal process has been started but hasn't completed.
@@ -667,10 +673,12 @@ class LaunchpadBrowserPublication(
                     traversal_thread_duration)
             # Update statsd, timing is in milliseconds
             getUtility(IStatsdClient).timing(
-                'traversal_duration,success=False,pageid={}'.format(
-                    self._prepPageIDForMetrics(
-                        request._orig_env.get('launchpad.pageid'))
-                ), traversal_duration * 1000)
+                'traversal_duration', traversal_duration * 1000,
+                labels={
+                    'success': False,
+                    'pageid': self._prepPageIDForMetrics(
+                        request._orig_env.get('launchpad.pageid')),
+                    })
         else:
             # The exception wasn't raised in the middle of the traversal nor
             # the publication, so there's nothing we need to do here.
diff --git a/lib/lp/services/webapp/tests/test_publication.py b/lib/lp/services/webapp/tests/test_publication.py
index 60c7d94..97d67ad 100644
--- a/lib/lp/services/webapp/tests/test_publication.py
+++ b/lib/lp/services/webapp/tests/test_publication.py
@@ -320,12 +320,12 @@ class TestPublisherStats(StatsMixin, TestCaseWithFactory):
             [x[0] for x in self.stats_client.timing.call_args_list],
             MatchesListwise(
                 [MatchesListwise(
-                    (Equals('traversal_duration,success=True,'
-                     'pageid=RootObject-index-html,env=test'),
+                    (Equals('traversal_duration,env=test,'
+                     'pageid=RootObject-index-html,success=True'),
                      GreaterThan(0))),
                  MatchesListwise(
-                     (Equals('publication_duration,success=True,'
-                      'pageid=RootObject-index-html,env=test'),
+                     (Equals('publication_duration,env=test,'
+                      'pageid=RootObject-index-html,success=True'),
                       GreaterThan(0)))]))
 
     def test_traversal_failure_stats(self):
@@ -344,8 +344,8 @@ class TestPublisherStats(StatsMixin, TestCaseWithFactory):
             [x[0] for x in self.stats_client.timing.call_args_list],
             MatchesListwise(
                 [MatchesListwise(
-                    (Equals('traversal_duration,success=False,'
-                     'pageid=None,env=test'),
+                    (Equals('traversal_duration,env=test,'
+                     'pageid=None,success=False'),
                      GreaterThan(0)))]))
 
     def test_publication_failure_stats(self):
@@ -364,12 +364,12 @@ class TestPublisherStats(StatsMixin, TestCaseWithFactory):
             [x[0] for x in self.stats_client.timing.call_args_list],
             MatchesListwise(
                 [MatchesListwise(
-                    (Equals('traversal_duration,success=True,'
-                     'pageid=RootObject-index-html,env=test'),
+                    (Equals('traversal_duration,env=test,'
+                     'pageid=RootObject-index-html,success=True'),
                      GreaterThan(0))),
                  MatchesListwise(
-                     (Equals('publication_duration,success=False,'
-                      'pageid=RootObject-index-html,env=test'),
+                     (Equals('publication_duration,env=test,'
+                      'pageid=RootObject-index-html,success=False'),
                       GreaterThan(0)))]))
 
     def test_prepPageIDForMetrics_none(self):
diff --git a/lib/lp/snappy/tests/test_snapbuildbehaviour.py b/lib/lp/snappy/tests/test_snapbuildbehaviour.py
index dceaa04..0ace53a 100644
--- a/lib/lp/snappy/tests/test_snapbuildbehaviour.py
+++ b/lib/lp/snappy/tests/test_snapbuildbehaviour.py
@@ -784,7 +784,7 @@ class TestAsyncSnapBuildBehaviour(StatsMixin, TestSnapBuildBehaviourBase):
         self.assertEqual(1, self.stats_client.incr.call_count)
         self.assertEqual(
             self.stats_client.incr.call_args_list[0][0],
-            ('build.count,job_type=SNAPBUILD,builder_name={},env=test'.format(
+            ('build.count,builder_name={},env=test,job_type=SNAPBUILD'.format(
                 builder.name),))
 
     @defer.inlineCallbacks
diff --git a/lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py b/lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py
index 23aaf6e..6377868 100644
--- a/lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py
+++ b/lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py
@@ -187,8 +187,8 @@ class TestBinaryBuildPackageBehaviour(StatsMixin, TestCaseWithFactory):
         self.assertEqual(1, self.stats_client.incr.call_count)
         self.assertEqual(
             self.stats_client.incr.call_args_list[0][0],
-            ('build.count,job_type=PACKAGEBUILD,'
-             'builder_name={},env=test'.format(
+            ('build.count,builder_name={},env=test,'
+             'job_type=PACKAGEBUILD'.format(
                 builder.name),))
 
     @defer.inlineCallbacks
@@ -244,8 +244,8 @@ class TestBinaryBuildPackageBehaviour(StatsMixin, TestCaseWithFactory):
         self.assertEqual(1, self.stats_client.incr.call_count)
         self.assertEqual(
             self.stats_client.incr.call_args_list[0][0],
-            ('build.count,job_type=PACKAGEBUILD,'
-             'builder_name={},env=test'.format(
+            ('build.count,builder_name={},env=test,'
+             'job_type=PACKAGEBUILD'.format(
                 builder.name),))
 
     @defer.inlineCallbacks