sts-sponsors team mailing list archive

[Alexsander de Souza <mp+443706@xxxxxxxxxxxxxxxxxx>]

Alexsander de Souza has proposed merging ~alexsander-souza/maas-kpi/+git/maas-kpi:collect_github_stats into maas-kpi:master.

Commit message:
feat: collect stats from project hosted at GitHub

collect metrics about the number of bugs and PRs open


Requested reviews:
  MAAS Committers (maas-committers)

For more details, see:
https://code.launchpad.net/~alexsander-souza/maas-kpi/+git/maas-kpi/+merge/443706
-- 
Your team MAAS Committers is requested to review the proposed merge of ~alexsander-souza/maas-kpi/+git/maas-kpi:collect_github_stats into maas-kpi:master.

diff --git a/.gitignore b/.gitignore
index a2703c5..98fe27b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,4 +4,6 @@ build
 maaskpi.egg-info
 launchpad.creds
 influxdb.creds
+github.creds
 swift.key
+.tox
diff --git a/Makefile b/Makefile
index b1f2611..d6c104c 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,6 @@
 LP_CREDENTIALS ?= launchpad.creds
 SNAP_STORE_MACAROON ?= snapcraft.macaroon
+GH_CREDENTIALS ?= github.creds
 INFLUXDB_CREDENTIALS ?= influxdb.creds
 INFLUXDB_DBNAME ?= maas
 DAILYSTATS_DAYS ?= 1
@@ -65,11 +66,14 @@ $(GENERATED)/dailystats.metrics: | $(GENERATED) $(VIRTUALENV)
 $(GENERATED)/bugs.metrics: | $(GENERATED) $(VIRTUALENV)
 	$(VIRTUALENV)/bin/maaskpi-bugs -o $@ --lp-credentials $(LP_CREDENTIALS)
 
-metrics: $(GENERATED)/bugs.metrics $(GENERATED)/dailystats.metrics
+$(GENERATED)/github.metrics: | $(GENERATED) $(VIRTUALENV)
+	$(VIRTUALENV)/bin/maaskpi-gh -o $@ --gh-credentials $(GH_CREDENTIALS)
+
+metrics: $(GENERATED)/github.metrics $(GENERATED)/bugs.metrics $(GENERATED)/dailystats.metrics
 .PHONY: metrics
 
 # The push target needs the PUSH_GATEWAY variable to be defined.
-push: $(GENERATED)/bugs.metrics $(GENERATED)/dailystats.metrics | $(VIRTUALENV)
+push: $(GENERATED)/github.metrics $(GENERATED)/bugs.metrics $(GENERATED)/dailystats.metrics | $(VIRTUALENV)
 	for metric in $^ ; do \
 		echo "Pushing $$metric" ; \
 		$(VIRTUALENV)/bin/push-metrics --db $(INFLUXDB_DBNAME) --credentials $(INFLUXDB_CREDENTIALS) $(INFLUXDB_HOST) $$metric ; \
diff --git a/grafana/team.dashboard.py b/grafana/team.dashboard.py
index 71a0c2c..1ec2141 100644
--- a/grafana/team.dashboard.py
+++ b/grafana/team.dashboard.py
@@ -41,6 +41,32 @@ def untriaged_bugs_graph(title, project):
     )
 
 
+def github_graph(title, project):
+    return Graph(
+        title=title,
+        dataSource=get_datasource(),
+        targets=[
+            InfluxDBTarget(
+                query=(
+                    f"""
+                    SELECT "bug", "pr"
+                    FROM "maas.open_bugs"
+                    WHERE $timeFilter
+                      AND "project" = '{project}'
+                    """
+                ),
+            )
+        ],
+        yAxes=YAxes(YAxis(format=SHORT_FORMAT)),
+        stack=True,
+        nullPointMode=NULL_AS_ZERO,
+        tooltip=Tooltip(
+            sort=SORT_DESC,
+            valueType=INDIVIDUAL,
+        ),
+    )
+
+
 dashboard = Dashboard(
     title="MAAS Team",
     rows=[
@@ -49,7 +75,31 @@ dashboard = Dashboard(
                 untriaged_bugs_graph("Untriaged core bugs", "core"),
                 untriaged_bugs_graph("Untriaged UI bugs", "ui"),
             ]
-        )
+        ),
+        Row(
+            panels=[
+                github_graph("Packer-MAAS", "packer-maas"),
+                github_graph("Terraform Provider", "terraform-provider-maas"),
+            ]
+        ),
+        Row(
+            panels=[
+                github_graph("Prometheus Alert Rules", "maas-prometheus-alert-rules"),
+                github_graph("Loki Alert Rules", "maas-loki-alert-rules"),
+            ]
+        ),
+        Row(
+            panels=[
+                github_graph("Ansible Playbook", "maas-ansible-playbook"),
+                github_graph("Ansible Collection", "ansible-collection"),
+            ]
+        ),
+        Row(
+            panels=[
+                github_graph("Python libmaas", "python-libmaas"),
+                github_graph("Go MAAS client", "gomaasclient"),
+            ]
+        ),
     ],
     time=Time("now-30d", "now"),
     refresh=None,
diff --git a/maaskpi/github.py b/maaskpi/github.py
new file mode 100644
index 0000000..31b075e
--- /dev/null
+++ b/maaskpi/github.py
@@ -0,0 +1,70 @@
+from pathlib import Path
+
+from github import Github
+from influxdb import SeriesHelper
+
+from .base import Collector
+
+ORG_CANONICAL = "canonical"
+ORG_MAAS = "maas"
+
+
+class OpenBugsSeries(SeriesHelper):
+    class Meta:
+        series_name = "maas.open_bugs"
+        fields = ["bug", "pr"]
+        tags = ["project"]
+        autocommit = False
+
+
+class BugsCollector(Collector):
+    """Collect metrics about bugs against a GH project.
+
+    It collects the number of open bugs.
+
+    TODO use labels to identify 'triaged' bugs
+
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.parser.add_argument(
+            "-g",
+            "--gh-credentials",
+            default="github.creds",
+            nargs="?",
+            help="Path to the file containing GH API token",
+        )
+
+    def run_collect(self, args):
+        token = Path(args.gh_credentials).read_text().rstrip()
+        gh = Github(token)
+        return self.collect(gh)
+
+    def _collect_bugs(self, record_series, gh, org, project_label):
+        self.log(f"Tasks for {project_label}:")
+        counts = dict.fromkeys(record_series.Meta.fields, 0)
+        repo = gh.get_organization(org).get_repo(project_label)
+
+        for issue in repo.get_issues(state="open"):
+            kind = "pr" if issue.pull_request else "bug"
+            counts[kind] += 1
+            self.log(f"{project_label} | {issue.title} | {kind}")
+        # influx mutates state in the class via the constructor
+        record_series(**counts, project=project_label)
+
+    def collect(self, gh):
+        self._collect_bugs(OpenBugsSeries, gh, ORG_CANONICAL, "packer-maas")
+        self._collect_bugs(OpenBugsSeries, gh, ORG_CANONICAL, "maas-loki-alert-rules")
+        self._collect_bugs(
+            OpenBugsSeries, gh, ORG_CANONICAL, "maas-prometheus-alert-rules"
+        )
+        self._collect_bugs(OpenBugsSeries, gh, ORG_MAAS, "maas-ansible-playbook")
+        self._collect_bugs(OpenBugsSeries, gh, ORG_MAAS, "terraform-provider-maas")
+        self._collect_bugs(OpenBugsSeries, gh, ORG_MAAS, "ansible-collection")
+        self._collect_bugs(OpenBugsSeries, gh, ORG_MAAS, "python-libmaas")
+        self._collect_bugs(OpenBugsSeries, gh, ORG_MAAS, "gomaasclient")
+        yield OpenBugsSeries
+
+
+run = BugsCollector().run
diff --git a/pyproject.toml b/pyproject.toml
index acf21c9..11aeb2a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -13,6 +13,7 @@ dependencies = [
   "grafanalib",
   "influxdb",
   "launchpadlib",
+  "PyGithub",
   "pymacaroons",
   "python-keystoneclient",
   "python-swiftclient",
@@ -21,6 +22,7 @@ dependencies = [
 [project.scripts]
 maaskpi-bugs = "maaskpi.bugs:run"
 maaskpi-dailystats = "maaskpi.dailystats:run"
+maaskpi-gh = "maaskpi.github:run"
 maaskpi-snap = "maaskpi.snap:run"
 push-metrics = "maaskpi.influxdb:push_metrics"
 
diff --git a/requirements.txt b/requirements.txt
index ef8218b..5addca0 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,7 +2,9 @@ attrs==23.1.0
 certifi==2023.5.7
 cffi==1.15.1
 charset-normalizer==3.1.0
+cryptography==40.0.2
 debtcollector==2.5.0
+Deprecated==1.2.13
 distro==1.8.0
 grafanalib==0.7.0
 httplib2==0.22.0
@@ -25,6 +27,8 @@ oslo.utils==6.1.0
 packaging==23.1
 pbr==5.11.1
 pycparser==2.21
+PyGithub==1.58.2
+PyJWT==2.7.0
 pymacaroons==0.13.0
 PyNaCl==1.5.0
 pyparsing==3.0.9
@@ -33,7 +37,7 @@ python-keystoneclient==5.1.0
 python-swiftclient==4.3.0
 pytz==2023.3
 PyYAML==6.0
-requests==2.30.0
+requests==2.31.0
 rfc3986==2.0.0
 six==1.16.0
 stevedore==5.1.0