sts-sponsors team mailing list archive

[Björn Tillenius <mp+433519@xxxxxxxxxxxxxxxxxx>]

Björn Tillenius has proposed merging ~bjornt/maas:perf-test-no-custom-runner into maas:master with ~bjornt/maas:perf-test-dont-measure-setup as a prerequisite.

Commit message:
Remove the custom bin/test.perf runner.

There's no reason to have a custom test runner. It's better to
use pytest directly, so that it's clear how to use it.

I also improved the way we set the random seed, so that we do
it per-test and not per-session, and print out the used values
on test failures.

I kept bin/test.perf as a symlink, in case someone is very
used to it.



Requested reviews:
  MAAS Maintainers (maas-maintainers)

For more details, see:
https://code.launchpad.net/~bjornt/maas/+git/maas/+merge/433519
-- 
Your team MAAS Maintainers is requested to review the proposed merge of ~bjornt/maas:perf-test-no-custom-runner into maas:master.

diff --git a/Makefile b/Makefile
index 6ddf017..6f826b7 100644
--- a/Makefile
+++ b/Makefile
@@ -48,7 +48,6 @@ bin/subunit-1to2 \
 bin/subunit2junitxml \
 bin/subunit2pyunit \
 bin/test.parallel \
-bin/test.perf \
 bin/test.rack \
 bin/test.region \
 bin/test.region.legacy
@@ -62,7 +61,8 @@ build: \
   .run \
   $(VENV) \
   $(BIN_SCRIPTS) \
-  bin/py
+  bin/py \
+  bin/test.perf
 .PHONY: build
 
 all: build ui go-bins doc
@@ -100,6 +100,9 @@ $(BIN_SCRIPTS): $(VENV) bin
 bin/py: $(VENV) bin
 	ln -sf ../$(VENV)/bin/ipython3 $@
 
+bin/test.perf: $(VENV) bin
+	ln -sf ./pytest $@
+
 bin/database: bin/postgresfixture
 	ln -sf $(notdir $<) $@
 
@@ -127,16 +130,16 @@ test-py: bin/test.parallel bin/subunit-1to2 bin/subunit2junitxml bin/subunit2pyu
 	@utilities/run-py-tests-ci
 .PHONY: test-py
 
-test-perf: bin/test.perf
+test-perf: bin/pytest
 	GIT_BRANCH=$(shell git rev-parse --abbrev-ref HEAD) \
 	GIT_HASH=$(shell git rev-parse HEAD) \
-	bin/test.perf
+	bin/pytest src/maasperf/
 .PHONY: test-perf
 
 test-perf-quiet: bin/test.perf
 	GIT_BRANCH=$(shell git rev-parse --abbrev-ref HEAD) \
 	GIT_HASH=$(shell git rev-parse HEAD) \
-	bin/test.perf -q --disable-warnings --show-capture=no --no-header --no-summary
+	bin/pytest -q --disable-warnings --show-capture=no --no-header --no-summary src/maasperf/
 .PHONY: test-perf-quiet
 
 update-initial-sql: bin/database bin/maas-region cleandb
diff --git a/setup.cfg b/setup.cfg
index a849f7f..402ce94 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -37,7 +37,6 @@ console_scripts =
   test.region.legacy = maastesting.scripts:run_region_legacy
   test.rack = maastesting.scripts:run_rack
   test.parallel = maastesting.scripts:run_parallel
-  test.perf = maastesting.scripts:run_perf
 
 [options.packages.find]
 where = src
diff --git a/src/maasperf/tests/conftest.py b/src/maasperf/tests/conftest.py
index 5a784fa..02a24bb 100644
--- a/src/maasperf/tests/conftest.py
+++ b/src/maasperf/tests/conftest.py
@@ -7,14 +7,17 @@ from maasserver.models.user import get_auth_tokens
 from maasserver.testing.factory import factory as maasserver_factory
 from maasserver.testing.testclient import MAASSensibleOAuthClient
 from maastesting.perftest import perf
+from maastesting.pytest import configure_seeds, random_seed
 
 __all__ = [
     "admin_api_client",
     "api_client",
+    "configure_seeds",
     "django_db_setup",
     "factory",
     "maas_user",
     "perf",
+    "random_seed",
 ]
 
 
diff --git a/src/maastesting/perftest.py b/src/maastesting/perftest.py
index 435b854..01d1063 100644
--- a/src/maastesting/perftest.py
+++ b/src/maastesting/perftest.py
@@ -9,13 +9,10 @@ from cProfile import Profile
 from functools import wraps
 import json
 import os
-import random
 import sys
 import time
 
-from pytest import fixture
-from pytest import main as pytest_main
-from pytest import mark, skip
+from pytest import fixture, mark, skip
 
 from maastesting.fixtures import MAASDataFixture, MAASRootFixture
 
@@ -119,15 +116,6 @@ def perf_test(commit_transaction=False, db_only=False):
     return inner
 
 
-def run_perf_tests(env):
-    rand_seed = os.environ.get("MAAS_RAND_SEED")
-    random.seed(rand_seed)
-
-    cmd_args = sys.argv[1:]
-
-    pytest_main(args=cmd_args)
-
-
 @contextmanager
 def profile(testname: str, profiling_tag: str):
     """Produces profiling info for tests
diff --git a/src/maastesting/pytest.py b/src/maastesting/pytest.py
new file mode 100644
index 0000000..60e01cc
--- /dev/null
+++ b/src/maastesting/pytest.py
@@ -0,0 +1,30 @@
+# Copyright 2022 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+import os
+import random
+import time
+
+import pytest
+
+
+@pytest.fixture(scope="session")
+def configure_seeds():
+    maas_rand_seed = os.environ.get("MAAS_RAND_SEED")
+    if maas_rand_seed is None:
+        maas_rand_seed = time.time_ns()
+    python_hash_seed = os.environ.get("PYTHONHASHSEED")
+    if python_hash_seed is None:
+        python_hash_seed = random.randint(1, 4294967295)
+        os.environ["PYTHONHASHSEED"] = str(python_hash_seed)
+    return maas_rand_seed, python_hash_seed
+
+
+@pytest.fixture(autouse=True, scope="function")
+def random_seed(configure_seeds):
+    maas_rand_seed, python_hash_seed = configure_seeds
+    random.seed(maas_rand_seed)
+    yield
+    print(
+        f"MAAS_RAND_SEED={maas_rand_seed} "
+        f"PYTHONHASHSEED={python_hash_seed}",
+    )
diff --git a/utilities/run-perf-tests-ci b/utilities/run-perf-tests-ci
index 1fa17fb..2ddc8ab 100755
--- a/utilities/run-perf-tests-ci
+++ b/utilities/run-perf-tests-ci
@@ -21,7 +21,7 @@ echo "MAAS_RAND_SEED=${MAAS_RAND_SEED}"
 echo "PYTHONHASHSEED=${PYTHONHASHSEED}"
 
 bin/database --preserve run make syncdb || exit 1
-exec bin/database --preserve run -- bin/test.perf \
+exec bin/database --preserve run -- bin/pytest \
     -q \
     --disable-warnings \
     --show-capture=no \