sts-sponsors team mailing list archive

[Björn Tillenius <mp+443108@xxxxxxxxxxxxxxxxxx>]

Björn Tillenius has proposed merging ~bjornt/maas-kpi/+git/maas-kpi:remove-prometheus into maas-kpi:master.

Commit message:
Remove code related to Prometheus.

It was only used to migrate old data, and isn't needed anymore.

Also add a make target, update-py-deps, to update requirements.txt



Requested reviews:
  MAAS Committers (maas-committers)

For more details, see:
https://code.launchpad.net/~bjornt/maas-kpi/+git/maas-kpi/+merge/443108
-- 
Your team MAAS Committers is requested to review the proposed merge of ~bjornt/maas-kpi/+git/maas-kpi:remove-prometheus into maas-kpi:master.

diff --git a/Makefile b/Makefile
index 52e3ce1..ba048d5 100644
--- a/Makefile
+++ b/Makefile
@@ -26,6 +26,11 @@ deps:
 apps: $(VIRTUALENV)
 .PHONY: apps
 
+update-py-deps:
+	tox run -qq -e latest-requirements | \
+		grep -E -v '^(pkg-resources|-e|#)' > requirements.txt
+.PHONY: update-py-deps
+
 clean:
 	rm -rf $(GENERATED)
 	rm -rf .tox *.egg-info
diff --git a/maaskpi/base.py b/maaskpi/base.py
index 2c023b4..d8753fe 100644
--- a/maaskpi/base.py
+++ b/maaskpi/base.py
@@ -5,7 +5,6 @@ from contextlib import contextmanager
 from influxdb.line_protocol import make_lines
 from launchpadlib.launchpad import Launchpad
 from launchpadlib.uris import lookup_service_root
-from prometheus_client import CollectorRegistry
 
 
 @contextmanager
@@ -38,7 +37,6 @@ class Collector:
             default=sys.stdout.buffer,
             help="Path to the file where the metrics are written to",
         )
-        self.registry = CollectorRegistry()
 
     def log(self, msg):
         print(msg, file=sys.stderr)
diff --git a/maaskpi/prometheus.py b/maaskpi/prometheus.py
deleted file mode 100644
index fc957d4..0000000
--- a/maaskpi/prometheus.py
+++ /dev/null
@@ -1,105 +0,0 @@
-from datetime import datetime
-
-import requests
-
-from .base import Collector, get_and_reset
-from .bugs import OpenBugsSeries, status_to_field
-
-
-class PrometheusExporterBase(Collector):
-    """Export metrics from a Prometheus instance."""
-
-    def __init__(self):
-        super().__init__()
-        self.parser.add_argument(
-            "prometheus_host", help="Hostname of the Prometheus instance"
-        )
-        self.parser.add_argument("metrics_name", help="Name of the metric to export")
-        self.parser.add_argument("start", help="Start of the metrics export")
-        self.parser.add_argument(
-            "end", help="End of the metrics export", nargs="?", default=""
-        )
-        self.parser.add_argument(
-            "--step",
-            help="Timeseries step",
-            default="30m",
-            nargs="?",
-        )
-        self.parser.add_argument(
-            "--credentials",
-            default="influxdb.credentials",
-            nargs="?",
-            help="Path to the file containing InfluxDB credentials",
-        )
-        self.parser.add_argument(
-            "--port",
-            help="Port of the InfluxDB instance",
-            type=int,
-            default=8086,
-            nargs="?",
-        )
-
-    def run_collect(self, args):
-        if args.end:
-            end = args.end
-        else:
-            now = datetime.utcnow()
-            end = now.isoformat("T") + "Z"
-        params = {
-            "query": args.metrics_name,
-            "start": args.start,
-            "end": end,
-            "step": args.step,
-        }
-        req = requests.get(
-            f"http://{args.prometheus_host}:9090/api/v1/query_range";, params=params
-        )
-        data = req.json()
-        return self.collect(data)
-
-
-class OpenBugsPrometheusExporter(PrometheusExporterBase):
-    """This is a temporary class for migrating the old bugs metrics.
-
-    The migration has already been done, and it's only here to provide
-    an example of how migrating old Prometheus metrics may work.
-
-    We can remove this whole file after we've migrated everything to
-    influxdb.
-    """
-
-    def collect(self, data):
-        series_dict = {}
-        for result in data["data"]["result"]:
-            for timestamp, value in result["values"]:
-                timestamp = datetime.utcfromtimestamp(int(timestamp)).isoformat("T")
-                point_dict = series_dict.setdefault(timestamp, {})
-                value_dict = point_dict.setdefault(
-                    result["metric"]["project"], empty_point_dict()
-                )
-                value_dict[result["metric"]["status"]] = int(value)
-        for timestamp in sorted(series_dict.keys()):
-            point = series_dict[timestamp]
-            for project, values in point.items():
-                open_bugs = {}
-                for key, value in values.items():
-                    if key == "project":
-                        continue
-                    open_bugs[status_to_field(key)] = value
-                OpenBugsSeries(**open_bugs, project=project, time=timestamp)
-        with get_and_reset(OpenBugsSeries) as result:
-            return result
-
-
-def empty_point_dict():
-    return {
-        "New": 0,
-        "Confirmed": 0,
-        "Triaged": 0,
-        "IncompleteWithResponse": 0,
-        "IncompleteWithoutResponse": 0,
-        "Fix Committed": 0,
-    }
-
-
-prometheus_export = OpenBugsPrometheusExporter().run
diff --git a/pyproject.toml b/pyproject.toml
index b413de7..f145c4a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -14,7 +14,6 @@ dependencies = [
   "grafanalib",
   "influxdb",
   "launchpadlib",
-  "prometheus-client",
   "pymacaroons",
   "python-keystoneclient",
   "python-swiftclient",
@@ -24,7 +23,6 @@ dependencies = [
 maaskpi-bugs = "maaskpi.bugs:run"
 maaskpi-dailystats = "maaskpi.dailystats:run"
 maaskpi-snap = "maaskpi.snap:run"
-prometheus-export = "maaskpi.prometheus:prometheus_export"
 push-metrics = "maaskpi.influxdb:push_metrics"
 
 [tool.setuptools.packages.find]
diff --git a/requirements.txt b/requirements.txt
index f1b8e8b..4805aa1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,51 +1,43 @@
-attr==0.3.1
-attrs==21.2.0
-certifi==2021.5.30
-cffi==1.14.6
-charset-normalizer==2.0.4
-cryptography==3.4.8
-debtcollector==2.3.0
-distro==1.6.0
-grafanalib==0.5.13
-httplib2==0.19.1
-idna==3.2
-importlib-metadata==4.8.1
+attr==0.3.2
+attrs==23.1.0
+certifi==2023.5.7
+cffi==1.15.1
+charset-normalizer==3.1.0
+debtcollector==2.5.0
+distro==1.8.0
+grafanalib==0.7.0
+httplib2==0.22.0
+idna==3.4
 influxdb==5.3.1
-iso8601==0.1.16
-jeepney==0.7.1
-keyring==23.1.0
-keystoneauth1==4.3.1
-launchpadlib==1.10.13
-lazr.restfulclient==0.14.3
-lazr.uri==1.0.5
-msgpack==1.0.2
+iso8601==1.1.0
+keystoneauth1==5.2.0
+launchpadlib==1.11.0
+lazr.restfulclient==0.14.5
+lazr.uri==1.0.6
+msgpack==1.0.5
 netaddr==0.8.0
 netifaces==0.11.0
-oauthlib==3.1.1
+oauthlib==3.2.2
 os-service-types==1.7.0
-oslo.config==8.7.1
-oslo.i18n==5.1.0
-oslo.serialization==4.2.0
-oslo.utils==4.10.0
-packaging==21.0
-pbr==5.6.0
-prometheus-client==0.11.0
-pycparser==2.20
+oslo.config==9.1.1
+oslo.i18n==6.0.0
+oslo.serialization==5.1.1
+oslo.utils==6.1.0
+packaging==23.1
+pbr==5.11.1
+pycparser==2.21
 pymacaroons==0.13.0
-PyNaCl==1.4.0
-pyparsing==2.4.7
+PyNaCl==1.5.0
+pyparsing==3.0.9
 python-dateutil==2.8.2
-python-keystoneclient==4.2.0
-python-swiftclient==3.12.0
-pytz==2021.1
-PyYAML==5.4.1
-requests==2.26.0
-rfc3986==1.5.0
-SecretStorage==3.3.1
+python-keystoneclient==5.1.0
+python-swiftclient==4.3.0
+pytz==2023.3
+PyYAML==6.0
+requests==2.30.0
+rfc3986==2.0.0
 six==1.16.0
-stevedore==3.4.0
-testresources==2.0.1
-urllib3==1.26.6
-wadllib==1.3.5
-wrapt==1.12.1
-zipp==3.5.0
+stevedore==5.1.0
+urllib3==2.0.2
+wadllib==1.3.6
+wrapt==1.15.0
diff --git a/tox.ini b/tox.ini
index 4d98fad..8c7aaa6 100644
--- a/tox.ini
+++ b/tox.ini
@@ -26,6 +26,12 @@ deps =
     -e .
     -r requirements.txt
 
+[testenv:latest-requirements]
+deps =
+    -e .
+commands =
+    pip freeze
+
 [files]
 lint =
     grafana \