← Back to team overview

sts-sponsors team mailing list archive

[Merge] ~ack/maas-kpi:raw-queries into maas-kpi:master

 

Alberto Donato has proposed merging ~ack/maas-kpi:raw-queries into maas-kpi:master.

Commit message:
use raw queries in all dashboards

This also cleans up some unneded helpers for InfluxDBTarget, bumps codebase to Python3.10 and replaces attrs with dataclasses



Requested reviews:
  MAAS Committers (maas-committers)

For more details, see:
https://code.launchpad.net/~ack/maas-kpi/+git/maas-kpi/+merge/443168
-- 
Your team MAAS Committers is requested to review the proposed merge of ~ack/maas-kpi:raw-queries into maas-kpi:master.
diff --git a/Makefile b/Makefile
index ba048d5..b1f2611 100644
--- a/Makefile
+++ b/Makefile
@@ -27,7 +27,7 @@ apps: $(VIRTUALENV)
 .PHONY: apps
 
 update-py-deps:
-	tox run -qq -e latest-requirements | \
+	tox run -qq -r -e latest-requirements | \
 		grep -E -v '^(pkg-resources|-e|#)' > requirements.txt
 .PHONY: update-py-deps
 
diff --git a/grafana/features.dashboard.py b/grafana/features.dashboard.py
index f7803f7..1bfa471 100644
--- a/grafana/features.dashboard.py
+++ b/grafana/features.dashboard.py
@@ -41,7 +41,6 @@ dashboard = Dashboard(
                     "Machines",
                     [
                         InfluxDBTarget(
-                            rawQuery=True,
                             query=(
                                 """
                                 SELECT SUM("machines")
@@ -58,7 +57,6 @@ dashboard = Dashboard(
                     "Reporting deployments",
                     [
                         InfluxDBTarget(
-                            rawQuery=True,
                             query=(
                                 """
                                 SELECT SUM("deployment_count")
@@ -80,7 +78,6 @@ dashboard = Dashboard(
                     "Annotation Counts",
                     [
                         InfluxDBTarget(
-                            rawQuery=True,
                             query=(
                                 """
                                 SELECT
@@ -106,7 +103,6 @@ dashboard = Dashboard(
                     "Registered hosts",
                     [
                         InfluxDBTarget(
-                            rawQuery=True,
                             query=(
                                 """
                                 SELECT
@@ -127,7 +123,6 @@ dashboard = Dashboard(
                     "VMs",
                     [
                         InfluxDBTarget(
-                            rawQuery=True,
                             query=(
                                 """
                                 SELECT
@@ -148,7 +143,6 @@ dashboard = Dashboard(
                     "Cores",
                     [
                         InfluxDBTarget(
-                            rawQuery=True,
                             query=(
                                 """
                                 SELECT
@@ -171,7 +165,6 @@ dashboard = Dashboard(
                     "Memory",
                     [
                         InfluxDBTarget(
-                            rawQuery=True,
                             query=(
                                 """
                                 SELECT
@@ -201,7 +194,6 @@ dashboard = Dashboard(
                     "Power drivers",
                     [
                         InfluxDBTarget(
-                            rawQuery=True,
                             query=(
                                 """
                                 SELECT SUM("count")
diff --git a/grafana/kpis.dashboard.py b/grafana/kpis.dashboard.py
index 6e8020c..dd08095 100644
--- a/grafana/kpis.dashboard.py
+++ b/grafana/kpis.dashboard.py
@@ -61,7 +61,6 @@ dashboard = Dashboard(
                     "MAAS Regions",
                     [
                         InfluxDBTarget(
-                            rawQuery=True,
                             query=(
                                 """
                                 SELECT SUM("count")
@@ -79,7 +78,6 @@ dashboard = Dashboard(
                     "Machines",
                     [
                         InfluxDBTarget(
-                            rawQuery=True,
                             query=(
                                 """
                                 SELECT SUM("machines")
@@ -96,7 +94,6 @@ dashboard = Dashboard(
                     "Devices",
                     [
                         InfluxDBTarget(
-                            rawQuery=True,
                             query=(
                                 """
                                 SELECT SUM("devices")
@@ -113,7 +110,6 @@ dashboard = Dashboard(
                     "Controllers",
                     [
                         InfluxDBTarget(
-                            rawQuery=True,
                             query=(
                                 """
                                 SELECT SUM("regionrack_controllers")
@@ -125,7 +121,6 @@ dashboard = Dashboard(
                             alias="Total number of region+rack controllers",
                         ),
                         InfluxDBTarget(
-                            rawQuery=True,
                             query=(
                                 """
                                 SELECT SUM("region_controllers")
@@ -137,7 +132,6 @@ dashboard = Dashboard(
                             alias="Total number of region controllers",
                         ),
                         InfluxDBTarget(
-                            rawQuery=True,
                             query=(
                                 """
                                 SELECT SUM("rack_controllers")
@@ -160,7 +154,6 @@ dashboard = Dashboard(
                     "Major versions",
                     [
                         InfluxDBTarget(
-                            rawQuery=True,
                             query=(
                                 rf"""
                                 SELECT SUM("count") AS "{label}"
@@ -180,7 +173,6 @@ dashboard = Dashboard(
                     "Top versions",
                     [
                         InfluxDBTarget(
-                            rawQuery=True,
                             # XXX: This doesn't quite work. It seems to return
                             # the correct counts, but all versions are
                             # returned, not only top 10.
@@ -209,7 +201,6 @@ dashboard = Dashboard(
                     "Image downloads by architecture",
                     [
                         InfluxDBTarget(
-                            rawQuery=True,
                             query=(
                                 """
                                 SELECT SUM("count")
@@ -227,7 +218,6 @@ dashboard = Dashboard(
                     "Image downloads by series",
                     [
                         InfluxDBTarget(
-                            rawQuery=True,
                             query=(
                                 """
                                 SELECT SUM("count")
@@ -250,7 +240,6 @@ dashboard = Dashboard(
                     "IPv6",
                     [
                         InfluxDBTarget(
-                            rawQuery=True,
                             query=(
                                 """
                                 SELECT SUM("deployment_count")
@@ -268,7 +257,6 @@ dashboard = Dashboard(
                     "CentOS",
                     [
                         InfluxDBTarget(
-                            rawQuery=True,
                             query=(
                                 """
                                 SELECT SUM("count")
@@ -293,7 +281,6 @@ dashboard = Dashboard(
                     "MAAS deployments with over 1k machines",
                     [
                         InfluxDBTarget(
-                            rawQuery=True,
                             query=(
                                 """
                                 SELECT "machines"
@@ -315,7 +302,6 @@ dashboard = Dashboard(
                     "MAAS deployments by size",
                     [
                         InfluxDBTarget(
-                            rawQuery=True,
                             query=(
                                 f"""
                                 SELECT "count"
diff --git a/grafana/machine-listing-spike.dashboard.py b/grafana/machine-listing-spike.dashboard.py
index 655e854..9012b4a 100644
--- a/grafana/machine-listing-spike.dashboard.py
+++ b/grafana/machine-listing-spike.dashboard.py
@@ -11,6 +11,7 @@ from grafanalib.core import (
     SORT_DESC,
     Annotations,
     Dashboard,
+    Graph,
     Row,
     Template,
     Templating,
@@ -20,7 +21,7 @@ from grafanalib.core import (
     YAxis,
 )
 
-from maaskpi.grafana import Graph, InfluxDBTarget, Tag, TagField, get_datasource
+from maaskpi.grafana import InfluxDBTarget, get_datasource
 
 RESULTS_PREFIX = "maasspike"
 MACHINE_COUNTS = ("50", "all")
@@ -48,24 +49,19 @@ METRICS = (
 
 
 def create_metric_graph(machine_count, metric):
-    query = dedent(
-        f"""\
-        SELECT "{metric.name}" AS "{metric.label}"
-        FROM maas_ci_perf..testcase
-        WHERE "system" = \'$system\'
-          AND "dataset" =~ /^$dataset/
-          AND "test" =~ /{RESULTS_PREFIX}_.*_{machine_count}$/
-          AND $timeFilter
-        GROUP BY "dataset", "test"
-        """
-    )
     targets = [
         InfluxDBTarget(
-            measurement="testcase",
-            query=query,
-            rawQuery=True,
-            groupBy=[TagField("test"), TagField("dataset")],
-            tags=[Tag(key="system", operator="=", value="$system")],
+            query=(
+                f"""
+                SELECT "{metric.name}"
+                FROM maas_ci_perf..testcase
+                WHERE $timeFilter
+                  AND "system" = \'$system\'
+                  AND "dataset" =~ /^$dataset/
+                  AND "test" =~ /{RESULTS_PREFIX}_.*_{machine_count}$/
+                GROUP BY "dataset", "test"
+                """
+            ),
             alias="$tag_test - $tag_dataset",
         )
     ]
@@ -89,24 +85,19 @@ def create_handler_graph(handler, machines_count, metrics):
     targets = []
     yaxes_formats = []
     for metric in metrics:
-        query = dedent(
-            f"""\
-            SELECT "{metric.name}" AS "{metric.label}"
-            FROM maas_ci_perf..testcase
-            WHERE "system" = \'$system\'
-              AND "dataset" =~ /^$dataset/
-              AND "test" =~ /{RESULTS_PREFIX}_{handler}_/
-              AND $timeFilter
-            GROUP BY "dataset"
-            """
-        )
         targets.append(
             InfluxDBTarget(
-                measurement="testcase",
-                query=query,
-                rawQuery=True,
-                groupBy=[TagField("dataset")],
-                tags=[Tag(key="system", operator="=", value="$system")],
+                query=(
+                    f"""
+                    SELECT "{metric.name}"
+                    FROM maas_ci_perf..testcase
+                    WHERE $timeFilter
+                      AND "system" = \'$system\'
+                      AND "dataset" =~ /^$dataset/
+                      AND "test" =~ /{RESULTS_PREFIX}_{handler}_/
+                    GROUP BY "dataset"
+                    """
+                ),
                 alias=f"{metric.label} - $tag_dataset",
             )
         )
@@ -156,15 +147,6 @@ def create_rows():
 
 
 def create_annotations():
-    query = dedent(
-        """\
-        SELECT "duration", "revision", "dataset"
-        FROM maas_ci_perf..testcase
-        WHERE "dataset" =~ /^$dataset/
-          AND $timeFilter
-        ORDER BY time ASC
-        """
-    )
     return Annotations(
         list=[
             {
@@ -172,7 +154,15 @@ def create_annotations():
                 "enable": True,
                 "hide": False,
                 "name": "git",
-                "query": query,
+                "query": dedent(
+                    """
+                    SELECT "duration", "revision", "dataset"
+                    FROM maas_ci_perf..testcase
+                    WHERE $timeFilter
+                      AND "dataset" =~ /^$dataset/
+                    ORDER BY time ASC
+                    """
+                ).strip(),
                 "tagsColumn": "dataset",
                 "textColumn": "revision",
                 "type": "tags",
diff --git a/grafana/performance.dashboard.py b/grafana/performance.dashboard.py
index 87e255a..d8fd31d 100644
--- a/grafana/performance.dashboard.py
+++ b/grafana/performance.dashboard.py
@@ -1,5 +1,7 @@
 # Copyright 2022 Canonical Ltd.  This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
+# GNU Affero General Public License version 3 (see the file LICENSE)
+
+from textwrap import dedent
 
 from grafanalib.core import (
     INDIVIDUAL,
@@ -8,6 +10,7 @@ from grafanalib.core import (
     SORT_DESC,
     Annotations,
     Dashboard,
+    Graph,
     Row,
     Template,
     Templating,
@@ -17,7 +20,7 @@ from grafanalib.core import (
     YAxis,
 )
 
-from maaskpi.grafana import Graph, InfluxDBTarget, Tag, TagField, get_datasource
+from maaskpi.grafana import InfluxDBTarget, get_datasource
 
 perf_tests = {
     "test_perf_create_machines": {
@@ -56,14 +59,18 @@ perf_tests = {
 def create_graph(title, testcase, **kwargs):
     targets = [
         InfluxDBTarget(
-            measurement="testcase",
-            query=f'SELECT "duration" AS "Test duration" FROM maas_ci_perf..testcase WHERE "system" =~ /^$system$/ AND "dataset" =~ /^$dataset/ AND "test" = \'{testcase}\' AND $timeFilter GROUP BY "dataset"',
-            rawQuery=True,
-            groupBy=[TagField("dataset")],
-            tags=[
-                Tag(key="system", operator="=~", value="/^$system$/"),
-            ],
-            alias="$tag_dataset",
+            query=(
+                f"""
+                SELECT "duration"
+                FROM maas_ci_perf..testcase
+                WHERE $timeFilter
+                  AND "system" =~ /^$system$/
+                  AND "dataset" =~ /^$dataset/
+                  AND "test" = '{testcase}'
+                GROUP BY "dataset"
+                """
+            ),
+            alias="Test duration",
         )
     ]
 
@@ -103,7 +110,15 @@ def create_annotations():
                 "enable": True,
                 "hide": False,
                 "name": "git",
-                "query": 'SELECT "duration","revision","dataset" from maas_ci_perf..testcase WHERE "dataset" =~ /^$dataset/ AND $timeFilter ORDER BY time ASC',
+                "query": dedent(
+                    """
+                    SELECT "duration", "revision", "dataset"
+                    FROM maas_ci_perf..testcase
+                    WHERE $timeFilter
+                      AND "dataset" =~ /^$dataset/
+                    ORDER BY time ASC
+                    """
+                ).strip(),
                 "tagsColumn": "dataset",
                 "textColumn": "revision",
                 "type": "tags",
diff --git a/grafana/team.dashboard.py b/grafana/team.dashboard.py
index 04a6a5a..1144579 100644
--- a/grafana/team.dashboard.py
+++ b/grafana/team.dashboard.py
@@ -1,4 +1,3 @@
-import attr
 from grafanalib.core import (
     INDIVIDUAL,
     NULL_AS_ZERO,
@@ -13,15 +12,28 @@ from grafanalib.core import (
     YAxis,
 )
 
-from maaskpi.grafana import Field, InfluxDBTarget, Select, Tag, get_datasource
+from maaskpi.grafana import InfluxDBTarget, get_datasource
 
 
-def stacked(graph):
-    """Turn a graph into a stacked graph."""
-    return attr.assoc(
-        graph,
-        nullPointMode=NULL_AS_ZERO,
+def untriaged_bugs_graph(title, project):
+    return Graph(
+        title=title,
+        dataSource=get_datasource(),
+        targets=[
+            InfluxDBTarget(
+                query=(
+                    """
+                    SELECT "confirmed", "incompletewithresponse", "new"
+                    FROM "maas.open_bugs"
+                    WHERE $timeFilter
+                      AND "project" = 'core'
+                    """
+                ),
+            )
+        ],
+        yAxes=YAxes(YAxis(format=SHORT_FORMAT)),
         stack=True,
+        nullPointMode=NULL_AS_ZERO,
         tooltip=Tooltip(
             sort=SORT_DESC,
             valueType=INDIVIDUAL,
@@ -29,30 +41,6 @@ def stacked(graph):
     )
 
 
-def untriaged_bugs_graph(title, project):
-    return stacked(
-        Graph(
-            title=title,
-            dataSource=get_datasource(),
-            targets=[
-                InfluxDBTarget(
-                    measurement="maas.open_bugs",
-                    select=Select(
-                        [
-                            [Field("confirmed")],
-                            [Field("incompletewithresponse")],
-                            [Field("new")],
-                        ]
-                    ),
-                    tags=[Tag("project", "=", project)],
-                )
-            ],
-            stack=True,
-            yAxes=YAxes(YAxis(format=SHORT_FORMAT)),
-        )
-    )
-
-
 dashboard = Dashboard(
     title="MAAS Team",
     rows=[
diff --git a/maaskpi/dailystats.py b/maaskpi/dailystats.py
index d3adc89..56f5526 100644
--- a/maaskpi/dailystats.py
+++ b/maaskpi/dailystats.py
@@ -7,7 +7,6 @@ from collections import Counter, defaultdict
 from dataclasses import asdict, dataclass, field, fields
 from datetime import date, datetime, timedelta
 from pathlib import Path
-from typing import Optional, Set
 from urllib.parse import parse_qs, urlparse
 
 import swiftclient
@@ -338,7 +337,7 @@ class DeploymentSizeBucket:
     """A bucket counting number of deployments by machine size."""
 
     min: int
-    max: Optional[int] = None
+    max: int | None = None
     count: int = 0
     tag: str = field(init=False)
     message: str = field(init=False)
@@ -502,14 +501,14 @@ class DailyStats:
                 count=bucket.count,
             )
 
-    def _get_top_deployments(self) -> Set[str]:
+    def _get_top_deployments(self) -> set[str]:
         counter = Counter(
             {
                 uuid: info["stats"].machines if "stats" in info else 0
                 for uuid, info in self.entries.items()
             }
         )
-        return set(uuid for uuid, _ in counter.most_common(self.top_deploys_count))
+        return {uuid for uuid, _ in counter.most_common(self.top_deploys_count)}
 
     def _update_deployments_size_count(self, info):
         stats = info.get("stats")
@@ -690,7 +689,7 @@ class DailyStatsCollector(Collector):
 
     def run_collect(self, args):
         if args.swift_key:
-            with open(args.swift_key, "r") as swift_key_file:
+            with open(args.swift_key) as swift_key_file:
                 swift_key = swift_key_file.read().strip()
             swift = swiftclient.Connection(
                 authurl=AUTH_URL,
diff --git a/maaskpi/grafana.py b/maaskpi/grafana.py
index 2dd7d32..45b6b27 100644
--- a/maaskpi/grafana.py
+++ b/maaskpi/grafana.py
@@ -1,8 +1,6 @@
 import os
 import textwrap
-
-import attr
-from grafanalib.core import Graph as GrafanaGraph
+from dataclasses import dataclass, field
 
 
 def get_datasource():
@@ -19,120 +17,19 @@ def get_datasource():
 TIME_SERIES_TARGET_FORMAT = "time_series"
 
 
-@attr.s
-class InfluxDBTarget(object):
-    select = attr.ib(default=None)
-    format = attr.ib(default=TIME_SERIES_TARGET_FORMAT)
-    measurement = attr.ib(default="")
-    orderByTime = attr.ib(default="ASC")
-    refId = attr.ib(default="A")
-    policy = attr.ib(default="default")
-    tags = attr.ib(default=attr.Factory(list))
-    groupBy = attr.ib(default=attr.Factory(list))
-    alias = attr.ib(default=None)
-    rawQuery = attr.ib(default=False)
-    query = attr.ib(default="")
+@dataclass
+class InfluxDBTarget:
+    format: str = field(default=TIME_SERIES_TARGET_FORMAT)
+    refId: str = field(default="A")
+    policy: str = field(default="default")
+    alias: str | None = field(default=None)
+    query: str = field(default="")
 
     def to_json_data(self):
-        data = {
-            "groupBy": self.groupBy,
+        return {
             "resultFormat": self.format,
-            "measurement": self.measurement,
             "refId": self.refId,
-            "tags": [tag.to_json_data() for tag in self.tags],
             "alias": self.alias,
+            "rawQuery": True,
+            "query": textwrap.dedent(self.query).strip(),
         }
-        if self.rawQuery:
-            data["rawQuery"] = self.rawQuery
-            data["query"] = textwrap.dedent(self.query).strip()
-        else:
-            data["select"] = self.select.to_json_data()
-        return data
-
-
-@attr.s
-class Select(object):
-    items = attr.ib(default=attr.Factory(list))
-
-    def to_json_data(self):
-        result = []
-        for item in self.items:
-            result.append([sub_item.to_json_data() for sub_item in item])
-        return result
-
-
-@attr.s
-class Field(object):
-    name = attr.ib(default="")
-
-    def to_json_data(self):
-        return {"params": [self.name], "type": "field"}
-
-
-@attr.s
-class TagField(object):
-    name = attr.ib(default="")
-
-    def to_json_data(self):
-        return {"params": [self.name], "type": "tag"}
-
-
-@attr.s
-class Tag(object):
-    key = attr.ib()
-    operator = attr.ib()
-    value = attr.ib()
-    condition = attr.ib(default=None)
-
-    def to_json_data(self):
-        data = {"key": self.key, "operator": self.operator, "value": self.value}
-        if self.condition:
-            data["condition"] = self.condition
-        return data
-
-
-@attr.s
-class Time(object):
-    interval = attr.ib(default="$__interval")
-
-    def to_json_data(self):
-        return {"type": "time", "params": [self.interval]}
-
-
-class Count(object):
-    def to_json_data(self):
-        return {"type": "count", "params": []}
-
-
-class Sum(object):
-    def to_json_data(self):
-        return {"type": "sum", "params": []}
-
-
-@attr.s
-class Alias(object):
-    name = attr.ib()
-
-    def to_json_data(self):
-        return {"type": "alias", "params": [self.name]}
-
-
-@attr.s
-class Transform(object):
-    """Transformations are a Grafana 8.x feature"""
-
-    id = attr.ib()
-    options = attr.ib(default=attr.Factory(dict))
-
-    def to_json_data(self):
-        return {"id": self.id, "options": self.options}
-
-
-@attr.s
-class Graph(GrafanaGraph):
-    transformations = attr.ib(default=attr.Factory(list))
-
-    def to_json_data(self):
-        data = super().to_json_data()
-        data["transformations"] = [item.to_json_data() for item in self.transformations]
-        return data
diff --git a/maaskpi/influxdb.py b/maaskpi/influxdb.py
index 1c8cf21..43c99eb 100644
--- a/maaskpi/influxdb.py
+++ b/maaskpi/influxdb.py
@@ -36,7 +36,7 @@ class InfluxDBPusher:
         args = self.parser.parse_args()
         with open(args.metrics_path, "rb") as metrics_file:
             data = metrics_file.read().decode("utf-8")
-        with open(args.credentials, "r") as credentials_file:
+        with open(args.credentials) as credentials_file:
             username, password = credentials_file.read().strip().split(" ", 1)
         client = InfluxDBClient(
             host=args.influxdb_host,
diff --git a/pyproject.toml b/pyproject.toml
index f145c4a..acf21c9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -10,7 +10,6 @@ version = "1.0"
 description = "Collect MAAS project KPIs"
 readme = "README.md"
 dependencies = [
-  "attr",
   "grafanalib",
   "influxdb",
   "launchpadlib",
diff --git a/requirements.txt b/requirements.txt
index 4805aa1..ef8218b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,3 @@
-attr==0.3.2
 attrs==23.1.0
 certifi==2023.5.7
 cffi==1.15.1

Follow ups