sts-sponsors team mailing list archive
-
sts-sponsors team
-
Mailing list archive
-
Message #08497
[Merge] ~ack/maas-kpi:raw-queries into maas-kpi:master
Alberto Donato has proposed merging ~ack/maas-kpi:raw-queries into maas-kpi:master.
Commit message:
use raw queries in all dashboards
This also cleans up some unneded helpers for InfluxDBTarget, bumps codebase to Python3.10 and replaces attrs with dataclasses
Requested reviews:
MAAS Committers (maas-committers)
For more details, see:
https://code.launchpad.net/~ack/maas-kpi/+git/maas-kpi/+merge/443168
--
Your team MAAS Committers is requested to review the proposed merge of ~ack/maas-kpi:raw-queries into maas-kpi:master.
diff --git a/Makefile b/Makefile
index ba048d5..b1f2611 100644
--- a/Makefile
+++ b/Makefile
@@ -27,7 +27,7 @@ apps: $(VIRTUALENV)
.PHONY: apps
update-py-deps:
- tox run -qq -e latest-requirements | \
+ tox run -qq -r -e latest-requirements | \
grep -E -v '^(pkg-resources|-e|#)' > requirements.txt
.PHONY: update-py-deps
diff --git a/grafana/features.dashboard.py b/grafana/features.dashboard.py
index f7803f7..1bfa471 100644
--- a/grafana/features.dashboard.py
+++ b/grafana/features.dashboard.py
@@ -41,7 +41,6 @@ dashboard = Dashboard(
"Machines",
[
InfluxDBTarget(
- rawQuery=True,
query=(
"""
SELECT SUM("machines")
@@ -58,7 +57,6 @@ dashboard = Dashboard(
"Reporting deployments",
[
InfluxDBTarget(
- rawQuery=True,
query=(
"""
SELECT SUM("deployment_count")
@@ -80,7 +78,6 @@ dashboard = Dashboard(
"Annotation Counts",
[
InfluxDBTarget(
- rawQuery=True,
query=(
"""
SELECT
@@ -106,7 +103,6 @@ dashboard = Dashboard(
"Registered hosts",
[
InfluxDBTarget(
- rawQuery=True,
query=(
"""
SELECT
@@ -127,7 +123,6 @@ dashboard = Dashboard(
"VMs",
[
InfluxDBTarget(
- rawQuery=True,
query=(
"""
SELECT
@@ -148,7 +143,6 @@ dashboard = Dashboard(
"Cores",
[
InfluxDBTarget(
- rawQuery=True,
query=(
"""
SELECT
@@ -171,7 +165,6 @@ dashboard = Dashboard(
"Memory",
[
InfluxDBTarget(
- rawQuery=True,
query=(
"""
SELECT
@@ -201,7 +194,6 @@ dashboard = Dashboard(
"Power drivers",
[
InfluxDBTarget(
- rawQuery=True,
query=(
"""
SELECT SUM("count")
diff --git a/grafana/kpis.dashboard.py b/grafana/kpis.dashboard.py
index 6e8020c..dd08095 100644
--- a/grafana/kpis.dashboard.py
+++ b/grafana/kpis.dashboard.py
@@ -61,7 +61,6 @@ dashboard = Dashboard(
"MAAS Regions",
[
InfluxDBTarget(
- rawQuery=True,
query=(
"""
SELECT SUM("count")
@@ -79,7 +78,6 @@ dashboard = Dashboard(
"Machines",
[
InfluxDBTarget(
- rawQuery=True,
query=(
"""
SELECT SUM("machines")
@@ -96,7 +94,6 @@ dashboard = Dashboard(
"Devices",
[
InfluxDBTarget(
- rawQuery=True,
query=(
"""
SELECT SUM("devices")
@@ -113,7 +110,6 @@ dashboard = Dashboard(
"Controllers",
[
InfluxDBTarget(
- rawQuery=True,
query=(
"""
SELECT SUM("regionrack_controllers")
@@ -125,7 +121,6 @@ dashboard = Dashboard(
alias="Total number of region+rack controllers",
),
InfluxDBTarget(
- rawQuery=True,
query=(
"""
SELECT SUM("region_controllers")
@@ -137,7 +132,6 @@ dashboard = Dashboard(
alias="Total number of region controllers",
),
InfluxDBTarget(
- rawQuery=True,
query=(
"""
SELECT SUM("rack_controllers")
@@ -160,7 +154,6 @@ dashboard = Dashboard(
"Major versions",
[
InfluxDBTarget(
- rawQuery=True,
query=(
rf"""
SELECT SUM("count") AS "{label}"
@@ -180,7 +173,6 @@ dashboard = Dashboard(
"Top versions",
[
InfluxDBTarget(
- rawQuery=True,
# XXX: This doesn't quite work. It seems to return
# the correct counts, but all versions are
# returned, not only top 10.
@@ -209,7 +201,6 @@ dashboard = Dashboard(
"Image downloads by architecture",
[
InfluxDBTarget(
- rawQuery=True,
query=(
"""
SELECT SUM("count")
@@ -227,7 +218,6 @@ dashboard = Dashboard(
"Image downloads by series",
[
InfluxDBTarget(
- rawQuery=True,
query=(
"""
SELECT SUM("count")
@@ -250,7 +240,6 @@ dashboard = Dashboard(
"IPv6",
[
InfluxDBTarget(
- rawQuery=True,
query=(
"""
SELECT SUM("deployment_count")
@@ -268,7 +257,6 @@ dashboard = Dashboard(
"CentOS",
[
InfluxDBTarget(
- rawQuery=True,
query=(
"""
SELECT SUM("count")
@@ -293,7 +281,6 @@ dashboard = Dashboard(
"MAAS deployments with over 1k machines",
[
InfluxDBTarget(
- rawQuery=True,
query=(
"""
SELECT "machines"
@@ -315,7 +302,6 @@ dashboard = Dashboard(
"MAAS deployments by size",
[
InfluxDBTarget(
- rawQuery=True,
query=(
f"""
SELECT "count"
diff --git a/grafana/machine-listing-spike.dashboard.py b/grafana/machine-listing-spike.dashboard.py
index 655e854..9012b4a 100644
--- a/grafana/machine-listing-spike.dashboard.py
+++ b/grafana/machine-listing-spike.dashboard.py
@@ -11,6 +11,7 @@ from grafanalib.core import (
SORT_DESC,
Annotations,
Dashboard,
+ Graph,
Row,
Template,
Templating,
@@ -20,7 +21,7 @@ from grafanalib.core import (
YAxis,
)
-from maaskpi.grafana import Graph, InfluxDBTarget, Tag, TagField, get_datasource
+from maaskpi.grafana import InfluxDBTarget, get_datasource
RESULTS_PREFIX = "maasspike"
MACHINE_COUNTS = ("50", "all")
@@ -48,24 +49,19 @@ METRICS = (
def create_metric_graph(machine_count, metric):
- query = dedent(
- f"""\
- SELECT "{metric.name}" AS "{metric.label}"
- FROM maas_ci_perf..testcase
- WHERE "system" = \'$system\'
- AND "dataset" =~ /^$dataset/
- AND "test" =~ /{RESULTS_PREFIX}_.*_{machine_count}$/
- AND $timeFilter
- GROUP BY "dataset", "test"
- """
- )
targets = [
InfluxDBTarget(
- measurement="testcase",
- query=query,
- rawQuery=True,
- groupBy=[TagField("test"), TagField("dataset")],
- tags=[Tag(key="system", operator="=", value="$system")],
+ query=(
+ f"""
+ SELECT "{metric.name}"
+ FROM maas_ci_perf..testcase
+ WHERE $timeFilter
+ AND "system" = \'$system\'
+ AND "dataset" =~ /^$dataset/
+ AND "test" =~ /{RESULTS_PREFIX}_.*_{machine_count}$/
+ GROUP BY "dataset", "test"
+ """
+ ),
alias="$tag_test - $tag_dataset",
)
]
@@ -89,24 +85,19 @@ def create_handler_graph(handler, machines_count, metrics):
targets = []
yaxes_formats = []
for metric in metrics:
- query = dedent(
- f"""\
- SELECT "{metric.name}" AS "{metric.label}"
- FROM maas_ci_perf..testcase
- WHERE "system" = \'$system\'
- AND "dataset" =~ /^$dataset/
- AND "test" =~ /{RESULTS_PREFIX}_{handler}_/
- AND $timeFilter
- GROUP BY "dataset"
- """
- )
targets.append(
InfluxDBTarget(
- measurement="testcase",
- query=query,
- rawQuery=True,
- groupBy=[TagField("dataset")],
- tags=[Tag(key="system", operator="=", value="$system")],
+ query=(
+ f"""
+ SELECT "{metric.name}"
+ FROM maas_ci_perf..testcase
+ WHERE $timeFilter
+ AND "system" = \'$system\'
+ AND "dataset" =~ /^$dataset/
+ AND "test" =~ /{RESULTS_PREFIX}_{handler}_/
+ GROUP BY "dataset"
+ """
+ ),
alias=f"{metric.label} - $tag_dataset",
)
)
@@ -156,15 +147,6 @@ def create_rows():
def create_annotations():
- query = dedent(
- """\
- SELECT "duration", "revision", "dataset"
- FROM maas_ci_perf..testcase
- WHERE "dataset" =~ /^$dataset/
- AND $timeFilter
- ORDER BY time ASC
- """
- )
return Annotations(
list=[
{
@@ -172,7 +154,15 @@ def create_annotations():
"enable": True,
"hide": False,
"name": "git",
- "query": query,
+ "query": dedent(
+ """
+ SELECT "duration", "revision", "dataset"
+ FROM maas_ci_perf..testcase
+ WHERE $timeFilter
+ AND "dataset" =~ /^$dataset/
+ ORDER BY time ASC
+ """
+ ).strip(),
"tagsColumn": "dataset",
"textColumn": "revision",
"type": "tags",
diff --git a/grafana/performance.dashboard.py b/grafana/performance.dashboard.py
index 87e255a..d8fd31d 100644
--- a/grafana/performance.dashboard.py
+++ b/grafana/performance.dashboard.py
@@ -1,5 +1,7 @@
# Copyright 2022 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
+# GNU Affero General Public License version 3 (see the file LICENSE)
+
+from textwrap import dedent
from grafanalib.core import (
INDIVIDUAL,
@@ -8,6 +10,7 @@ from grafanalib.core import (
SORT_DESC,
Annotations,
Dashboard,
+ Graph,
Row,
Template,
Templating,
@@ -17,7 +20,7 @@ from grafanalib.core import (
YAxis,
)
-from maaskpi.grafana import Graph, InfluxDBTarget, Tag, TagField, get_datasource
+from maaskpi.grafana import InfluxDBTarget, get_datasource
perf_tests = {
"test_perf_create_machines": {
@@ -56,14 +59,18 @@ perf_tests = {
def create_graph(title, testcase, **kwargs):
targets = [
InfluxDBTarget(
- measurement="testcase",
- query=f'SELECT "duration" AS "Test duration" FROM maas_ci_perf..testcase WHERE "system" =~ /^$system$/ AND "dataset" =~ /^$dataset/ AND "test" = \'{testcase}\' AND $timeFilter GROUP BY "dataset"',
- rawQuery=True,
- groupBy=[TagField("dataset")],
- tags=[
- Tag(key="system", operator="=~", value="/^$system$/"),
- ],
- alias="$tag_dataset",
+ query=(
+ f"""
+ SELECT "duration"
+ FROM maas_ci_perf..testcase
+ WHERE $timeFilter
+ AND "system" =~ /^$system$/
+ AND "dataset" =~ /^$dataset/
+ AND "test" = '{testcase}'
+ GROUP BY "dataset"
+ """
+ ),
+ alias="Test duration",
)
]
@@ -103,7 +110,15 @@ def create_annotations():
"enable": True,
"hide": False,
"name": "git",
- "query": 'SELECT "duration","revision","dataset" from maas_ci_perf..testcase WHERE "dataset" =~ /^$dataset/ AND $timeFilter ORDER BY time ASC',
+ "query": dedent(
+ """
+ SELECT "duration", "revision", "dataset"
+ FROM maas_ci_perf..testcase
+ WHERE $timeFilter
+ AND "dataset" =~ /^$dataset/
+ ORDER BY time ASC
+ """
+ ).strip(),
"tagsColumn": "dataset",
"textColumn": "revision",
"type": "tags",
diff --git a/grafana/team.dashboard.py b/grafana/team.dashboard.py
index 04a6a5a..1144579 100644
--- a/grafana/team.dashboard.py
+++ b/grafana/team.dashboard.py
@@ -1,4 +1,3 @@
-import attr
from grafanalib.core import (
INDIVIDUAL,
NULL_AS_ZERO,
@@ -13,15 +12,28 @@ from grafanalib.core import (
YAxis,
)
-from maaskpi.grafana import Field, InfluxDBTarget, Select, Tag, get_datasource
+from maaskpi.grafana import InfluxDBTarget, get_datasource
-def stacked(graph):
- """Turn a graph into a stacked graph."""
- return attr.assoc(
- graph,
- nullPointMode=NULL_AS_ZERO,
+def untriaged_bugs_graph(title, project):
+ return Graph(
+ title=title,
+ dataSource=get_datasource(),
+ targets=[
+ InfluxDBTarget(
+ query=(
+ """
+ SELECT "confirmed", "incompletewithresponse", "new"
+ FROM "maas.open_bugs"
+ WHERE $timeFilter
+ AND "project" = 'core'
+ """
+ ),
+ )
+ ],
+ yAxes=YAxes(YAxis(format=SHORT_FORMAT)),
stack=True,
+ nullPointMode=NULL_AS_ZERO,
tooltip=Tooltip(
sort=SORT_DESC,
valueType=INDIVIDUAL,
@@ -29,30 +41,6 @@ def stacked(graph):
)
-def untriaged_bugs_graph(title, project):
- return stacked(
- Graph(
- title=title,
- dataSource=get_datasource(),
- targets=[
- InfluxDBTarget(
- measurement="maas.open_bugs",
- select=Select(
- [
- [Field("confirmed")],
- [Field("incompletewithresponse")],
- [Field("new")],
- ]
- ),
- tags=[Tag("project", "=", project)],
- )
- ],
- stack=True,
- yAxes=YAxes(YAxis(format=SHORT_FORMAT)),
- )
- )
-
-
dashboard = Dashboard(
title="MAAS Team",
rows=[
diff --git a/maaskpi/dailystats.py b/maaskpi/dailystats.py
index d3adc89..56f5526 100644
--- a/maaskpi/dailystats.py
+++ b/maaskpi/dailystats.py
@@ -7,7 +7,6 @@ from collections import Counter, defaultdict
from dataclasses import asdict, dataclass, field, fields
from datetime import date, datetime, timedelta
from pathlib import Path
-from typing import Optional, Set
from urllib.parse import parse_qs, urlparse
import swiftclient
@@ -338,7 +337,7 @@ class DeploymentSizeBucket:
"""A bucket counting number of deployments by machine size."""
min: int
- max: Optional[int] = None
+ max: int | None = None
count: int = 0
tag: str = field(init=False)
message: str = field(init=False)
@@ -502,14 +501,14 @@ class DailyStats:
count=bucket.count,
)
- def _get_top_deployments(self) -> Set[str]:
+ def _get_top_deployments(self) -> set[str]:
counter = Counter(
{
uuid: info["stats"].machines if "stats" in info else 0
for uuid, info in self.entries.items()
}
)
- return set(uuid for uuid, _ in counter.most_common(self.top_deploys_count))
+ return {uuid for uuid, _ in counter.most_common(self.top_deploys_count)}
def _update_deployments_size_count(self, info):
stats = info.get("stats")
@@ -690,7 +689,7 @@ class DailyStatsCollector(Collector):
def run_collect(self, args):
if args.swift_key:
- with open(args.swift_key, "r") as swift_key_file:
+ with open(args.swift_key) as swift_key_file:
swift_key = swift_key_file.read().strip()
swift = swiftclient.Connection(
authurl=AUTH_URL,
diff --git a/maaskpi/grafana.py b/maaskpi/grafana.py
index 2dd7d32..45b6b27 100644
--- a/maaskpi/grafana.py
+++ b/maaskpi/grafana.py
@@ -1,8 +1,6 @@
import os
import textwrap
-
-import attr
-from grafanalib.core import Graph as GrafanaGraph
+from dataclasses import dataclass, field
def get_datasource():
@@ -19,120 +17,19 @@ def get_datasource():
TIME_SERIES_TARGET_FORMAT = "time_series"
-@attr.s
-class InfluxDBTarget(object):
- select = attr.ib(default=None)
- format = attr.ib(default=TIME_SERIES_TARGET_FORMAT)
- measurement = attr.ib(default="")
- orderByTime = attr.ib(default="ASC")
- refId = attr.ib(default="A")
- policy = attr.ib(default="default")
- tags = attr.ib(default=attr.Factory(list))
- groupBy = attr.ib(default=attr.Factory(list))
- alias = attr.ib(default=None)
- rawQuery = attr.ib(default=False)
- query = attr.ib(default="")
+@dataclass
+class InfluxDBTarget:
+ format: str = field(default=TIME_SERIES_TARGET_FORMAT)
+ refId: str = field(default="A")
+ policy: str = field(default="default")
+ alias: str | None = field(default=None)
+ query: str = field(default="")
def to_json_data(self):
- data = {
- "groupBy": self.groupBy,
+ return {
"resultFormat": self.format,
- "measurement": self.measurement,
"refId": self.refId,
- "tags": [tag.to_json_data() for tag in self.tags],
"alias": self.alias,
+ "rawQuery": True,
+ "query": textwrap.dedent(self.query).strip(),
}
- if self.rawQuery:
- data["rawQuery"] = self.rawQuery
- data["query"] = textwrap.dedent(self.query).strip()
- else:
- data["select"] = self.select.to_json_data()
- return data
-
-
-@attr.s
-class Select(object):
- items = attr.ib(default=attr.Factory(list))
-
- def to_json_data(self):
- result = []
- for item in self.items:
- result.append([sub_item.to_json_data() for sub_item in item])
- return result
-
-
-@attr.s
-class Field(object):
- name = attr.ib(default="")
-
- def to_json_data(self):
- return {"params": [self.name], "type": "field"}
-
-
-@attr.s
-class TagField(object):
- name = attr.ib(default="")
-
- def to_json_data(self):
- return {"params": [self.name], "type": "tag"}
-
-
-@attr.s
-class Tag(object):
- key = attr.ib()
- operator = attr.ib()
- value = attr.ib()
- condition = attr.ib(default=None)
-
- def to_json_data(self):
- data = {"key": self.key, "operator": self.operator, "value": self.value}
- if self.condition:
- data["condition"] = self.condition
- return data
-
-
-@attr.s
-class Time(object):
- interval = attr.ib(default="$__interval")
-
- def to_json_data(self):
- return {"type": "time", "params": [self.interval]}
-
-
-class Count(object):
- def to_json_data(self):
- return {"type": "count", "params": []}
-
-
-class Sum(object):
- def to_json_data(self):
- return {"type": "sum", "params": []}
-
-
-@attr.s
-class Alias(object):
- name = attr.ib()
-
- def to_json_data(self):
- return {"type": "alias", "params": [self.name]}
-
-
-@attr.s
-class Transform(object):
- """Transformations are a Grafana 8.x feature"""
-
- id = attr.ib()
- options = attr.ib(default=attr.Factory(dict))
-
- def to_json_data(self):
- return {"id": self.id, "options": self.options}
-
-
-@attr.s
-class Graph(GrafanaGraph):
- transformations = attr.ib(default=attr.Factory(list))
-
- def to_json_data(self):
- data = super().to_json_data()
- data["transformations"] = [item.to_json_data() for item in self.transformations]
- return data
diff --git a/maaskpi/influxdb.py b/maaskpi/influxdb.py
index 1c8cf21..43c99eb 100644
--- a/maaskpi/influxdb.py
+++ b/maaskpi/influxdb.py
@@ -36,7 +36,7 @@ class InfluxDBPusher:
args = self.parser.parse_args()
with open(args.metrics_path, "rb") as metrics_file:
data = metrics_file.read().decode("utf-8")
- with open(args.credentials, "r") as credentials_file:
+ with open(args.credentials) as credentials_file:
username, password = credentials_file.read().strip().split(" ", 1)
client = InfluxDBClient(
host=args.influxdb_host,
diff --git a/pyproject.toml b/pyproject.toml
index f145c4a..acf21c9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -10,7 +10,6 @@ version = "1.0"
description = "Collect MAAS project KPIs"
readme = "README.md"
dependencies = [
- "attr",
"grafanalib",
"influxdb",
"launchpadlib",
diff --git a/requirements.txt b/requirements.txt
index 4805aa1..ef8218b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,3 @@
-attr==0.3.2
attrs==23.1.0
certifi==2023.5.7
cffi==1.15.1
Follow ups
