sts-sponsors team mailing list archive

[Björn Tillenius <mp+433596@xxxxxxxxxxxxxxxxxx>]

Björn Tillenius has proposed merging ~bjornt/maas:perf-test-cli-options into maas:master with ~bjornt/maas:perf-test-no-custom-runner as a prerequisite.

Commit message:
Add pytest CLI options for maasperf.

Now --perf-output-file and --perf-profiling-tag can be specified
when running pytest to control where the output timeing and profiling
data.

I also fixed conftest.py so that pytest can process it in --help.
It had global imports of maasserver, which doesn't work since
django isn't set up when --help runs.

I also made use of pytest_plugins rather than importing all the
fixtures explictly.



Requested reviews:
  MAAS Lander (maas-lander): unittests
  MAAS Maintainers (maas-maintainers)

For more details, see:
https://code.launchpad.net/~bjornt/maas/+git/maas/+merge/433596
-- 
Your team MAAS Maintainers is requested to review the proposed merge of ~bjornt/maas:perf-test-cli-options into maas:master.

diff --git a/src/maasperf/tests/conftest.py b/src/maasperf/tests/conftest.py
index 02a24bb..6902a32 100644
--- a/src/maasperf/tests/conftest.py
+++ b/src/maasperf/tests/conftest.py
@@ -3,12 +3,15 @@
 
 from pytest import fixture
 
+<<<<<<< src/maasperf/tests/conftest.py
 from maasserver.models.user import get_auth_tokens
 from maasserver.testing.factory import factory as maasserver_factory
 from maasserver.testing.testclient import MAASSensibleOAuthClient
 from maastesting.perftest import perf
 from maastesting.pytest import configure_seeds, random_seed
 
+=======
+>>>>>>> src/maasperf/tests/conftest.py
 __all__ = [
     "admin_api_client",
     "api_client",
@@ -16,11 +19,17 @@ __all__ = [
     "django_db_setup",
     "factory",
     "maas_user",
+<<<<<<< src/maasperf/tests/conftest.py
     "perf",
     "random_seed",
+=======
+>>>>>>> src/maasperf/tests/conftest.py
 ]
 
 
+pytest_plugins = "maastesting.pytest.perftest,maastesting.pytest.seeds"
+
+
 # override pytest-django's db setup
 @fixture(scope="session")
 def django_db_setup():
@@ -29,6 +38,9 @@ def django_db_setup():
 
 @fixture(scope="session")
 def factory():
+    # Local imports from maasserver so that pytest --help works
+    from maasserver.testing.factory import factory as maasserver_factory
+
     return maasserver_factory
 
 
@@ -44,6 +56,10 @@ def maas_user(factory):
 
 @fixture()
 def api_client(maas_user):
+    # Local imports from maasserver so that pytest --help works
+    from maasserver.models.user import get_auth_tokens
+    from maasserver.testing.testclient import MAASSensibleOAuthClient
+
     return MAASSensibleOAuthClient(
         user=maas_user, token=get_auth_tokens(maas_user)[0]
     )
@@ -51,4 +67,8 @@ def api_client(maas_user):
 
 @fixture()
 def admin_api_client(admin):
+    # Local imports from maasserver so that pytest --help works
+    from maasserver.models.user import get_auth_tokens
+    from maasserver.testing.testclient import MAASSensibleOAuthClient
+
     return MAASSensibleOAuthClient(user=admin, token=get_auth_tokens(admin)[0])
diff --git a/src/maastesting/perftest.py b/src/maastesting/pytest/perftest.py
similarity index 85%
rename from src/maastesting/perftest.py
rename to src/maastesting/pytest/perftest.py
index 6c66950..a5469d4 100644
--- a/src/maastesting/perftest.py
+++ b/src/maastesting/pytest/perftest.py
@@ -32,13 +32,27 @@ def maas_data():
     return None
 
 
+def pytest_addoption(parser):
+    parser.addoption(
+        "--perf-output-file",
+        help="The file where to write the performance measurement as JSON.",
+    )
+    parser.addoption(
+        "--perf-profiling-tag",
+        help="If specified, create profiling dumps for the measured tests.",
+    )
+
+
 @fixture(scope="session")
-def perf():
+def perf(pytestconfig):
+    profiling_tag = pytestconfig.getoption("--perf-profiling-tag", None)
     perf_tester = PerfTester(
-        os.environ.get("GIT_BRANCH"), os.environ.get("GIT_HASH")
+        os.environ.get("GIT_BRANCH"),
+        os.environ.get("GIT_HASH"),
+        profiling_tag,
     )
     yield perf_tester
-    output = os.environ.get("OUTPUT_FILE")
+    output = pytestconfig.getoption("--perf-output-file", None)
     if output:
         with open(output, "w") as f:
             perf_tester.finish_build(f)
@@ -68,15 +82,15 @@ def measure_time():
 class PerfTester:
     """PerfTester is responsible for recording performance tests"""
 
-    def __init__(self, git_branch, git_hash):
+    def __init__(self, git_branch, git_hash, profiling_tag):
         self.results = {"branch": git_branch, "commit": git_hash, "tests": {}}
+        self.profiling_tag = profiling_tag
 
     @contextmanager
     def record(self, name):
         with ExitStack() as stack:
-            profiling_tag = os.environ.get("MAAS_PROFILING")
-            if profiling_tag:
-                stack.enter_context(profile(name, profiling_tag))
+            if self.profiling_tag:
+                stack.enter_context(profile(name, self.profiling_tag))
             timing = stack.enter_context(measure_time())
             yield
         self.results["tests"][name] = {"duration": timing.duration}
diff --git a/src/maastesting/pytest/seeds.py b/src/maastesting/pytest/seeds.py
new file mode 100644
index 0000000..012744b
--- /dev/null
+++ b/src/maastesting/pytest/seeds.py
@@ -0,0 +1,30 @@
+# Copyright 2022 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+import os
+import random
+import time
+
+import pytest
+
+
+@pytest.fixture(scope="session")
+def configure_seeds():
+    maas_rand_seed = os.environ.get("MAAS_RAND_SEED")
+    if maas_rand_seed is None:
+        maas_rand_seed = time.time_ns()
+    python_hash_seed = os.environ.get("PYTHONHASHSEED")
+    if python_hash_seed is None:
+        python_hash_seed = random.randint(1, 4294967295)
+        os.environ["PYTHONHASHSEED"] = str(python_hash_seed)
+    return maas_rand_seed, python_hash_seed
+
+
+@pytest.fixture(autouse=True)
+def random_seed(configure_seeds):
+    maas_rand_seed, python_hash_seed = configure_seeds
+    random.seed(maas_rand_seed)
+    yield
+    print(
+        f"MAAS_RAND_SEED={maas_rand_seed} "
+        f"PYTHONHASHSEED={python_hash_seed}",
+    )
diff --git a/utilities/run-perf-tests-ci b/utilities/run-perf-tests-ci
index 2ddc8ab..b312abb 100755
--- a/utilities/run-perf-tests-ci
+++ b/utilities/run-perf-tests-ci
@@ -15,7 +15,7 @@ MAAS_RAND_SEED="${MAAS_RAND_SEED:-$(od -vAn -N8 -tx8 < /dev/urandom | tr -d ' ')
 
 OUTPUT_FILE="${OUTPUT_FILE:-maas-perf-results.json}"
 
-export MAAS_RAND_SEED PYTHONHASHSEED GIT_HASH GIT_BRANCH OUTPUT_FILE
+export MAAS_RAND_SEED PYTHONHASHSEED GIT_HASH GIT_BRANCH
 
 echo "MAAS_RAND_SEED=${MAAS_RAND_SEED}"
 echo "PYTHONHASHSEED=${PYTHONHASHSEED}"
@@ -28,4 +28,5 @@ exec bin/database --preserve run -- bin/pytest \
     --no-header \
     --no-summary \
     --junit-xml=junit-perf.xml \
-    ./src/maasperf/
+    ./src/maasperf/ \
+    --perf-output-file ${OUTPUT_FILE}