testtools-dev team mailing list archive

[Robert Collins <robertc@xxxxxxxxxxxxxxxxx>]

Robert Collins has proposed merging lp:~lifeless/testtools/bug-1091512 into lp:testtools.

Requested reviews:
  testtools committers (testtools-committers)
Related bugs:
  Bug #1091512 in testtools: "load-list order is arbitrary"
  https://bugs.launchpad.net/testtools/+bug/1091512

For more details, see:
https://code.launchpad.net/~lifeless/testtools/bug-1091512/+merge/140369

This works around an annoying defect in the python discover implementation, giving more reliable test ordering for folk using discovery.

I took care to make it possible for things like testresources.OptimisingTestSuite to stay intact, though we still have an issue with --load-list and such suites. (bug 827175)
-- 
https://code.launchpad.net/~lifeless/testtools/bug-1091512/+merge/140369
Your team testtools developers is subscribed to branch lp:testtools.

=== modified file 'NEWS'
--- NEWS	2012-12-17 01:05:23 +0000
+++ NEWS	2012-12-18 09:13:25 +0000
@@ -6,6 +6,15 @@
 NEXT
 ~~~~
 
+Changes
+-------
+
+* ``testtools.run discover`` will now sort the tests it discovered. This is a 
+  workaround for http://bugs.python.org/issue16709. Non-standard test suites
+  are preserved, and their ``sort_tests()`` method called (if they have such an
+  attribute). ``testtools.testsuite.sorted_tests(suite, True)`` can be used by
+  such suites to do a local sort. (Robert Collins, #1091512)
+
 0.9.23
 ~~~~~~
 

=== modified file 'doc/for-framework-folk.rst'
--- doc/for-framework-folk.rst	2012-04-11 18:11:03 +0000
+++ doc/for-framework-folk.rst	2012-12-18 09:13:25 +0000
@@ -222,6 +222,17 @@
 it down after all of the tests are run. The fixture is *not* made available to
 any of the tests.
 
+sorted_tests
+------------
+
+Given the composite structure of TestSuite / TestCase, sorting tests is
+problematic - you can't tell what functionality is embedded into custom Suite
+implementations. In order to deliver consistent test orders when using test
+discovery (see http://bugs.python.org/issue16709), testtools flattens and
+sorts tests that have the standard TestSuite, defines a new method sort_tests,
+which can be used by non-standard TestSuites to know when they should sort
+their tests.
+
 .. _`testtools API docs`: http://mumak.net/testtools/apidocs/
 .. _unittest: http://docs.python.org/library/unittest.html
 .. _fixture: http://pypi.python.org/pypi/fixtures

=== modified file 'testtools/run.py'
--- testtools/run.py	2012-12-16 08:46:30 +0000
+++ testtools/run.py	2012-12-18 09:13:25 +0000
@@ -14,7 +14,7 @@
 
 from testtools import TextTestResult
 from testtools.compat import classtypes, istext, unicode_output_stream
-from testtools.testsuite import iterate_tests
+from testtools.testsuite import iterate_tests, sorted_tests
 
 
 defaultTestLoader = unittest.defaultTestLoader
@@ -75,6 +75,8 @@
 #  - --load-list has been added which can reduce the tests used (should be
 #    upstreamed).
 #  - The limitation of using getopt is declared to the user.
+#  - http://bugs.python.org/issue16709 is worked around, by sorting tests when
+#    discover is used.
 
 FAILFAST     = "  -f, --failfast   Stop on first failure\n"
 CATCHBREAK   = "  -c, --catch      Catch control-C and display results\n"
@@ -307,7 +309,17 @@
         top_level_dir = options.top
 
         loader = Loader()
-        self.test = loader.discover(start_dir, pattern, top_level_dir)
+        # See http://bugs.python.org/issue16709
+        # While sorting here is intrusive, its better than being random.
+        # Rules for the sort:
+        # - standard suites are flattened, and the resulting tests sorted by
+        #   id.
+        # - non-standard suites are preserved as-is, and sorted into position
+        #   by the first test found by iterating the suite.
+        # We do this by a DSU process: flatten and grab a key, sort, strip the
+        # keys.
+        loaded = loader.discover(start_dir, pattern, top_level_dir)
+        self.test = sorted_tests(loaded)
 
     def runTests(self):
         if (self.catchbreak

=== modified file 'testtools/tests/test_run.py'
--- testtools/tests/test_run.py	2012-12-17 00:57:44 +0000
+++ testtools/tests/test_run.py	2012-12-18 09:13:25 +0000
@@ -57,6 +57,27 @@
 testtools.runexample.TestFoo.test_quux
 """, out.getvalue())
 
+    def test_run_orders_tests(self):
+        self.useFixture(SampleTestFixture())
+        out = StringIO()
+        # We load two tests - one that exists and one that doesn't, and we
+        # should get the one that exists and neither the one that doesn't nor
+        # the unmentioned one that does.
+        tempdir = self.useFixture(fixtures.TempDir())
+        tempname = tempdir.path + '/tests.list'
+        f = open(tempname, 'wb')
+        try:
+            f.write(_b("""
+testtools.runexample.TestFoo.test_bar
+testtools.runexample.missingtest
+"""))
+        finally:
+            f.close()
+        run.main(['prog', '-l', '--load-list', tempname,
+            'testtools.runexample.test_suite'], out)
+        self.assertEqual("""testtools.runexample.TestFoo.test_bar
+""", out.getvalue())
+
     def test_run_load_list(self):
         self.useFixture(SampleTestFixture())
         out = StringIO()

=== modified file 'testtools/tests/test_testsuite.py'
--- testtools/tests/test_testsuite.py	2012-04-17 14:24:02 +0000
+++ testtools/tests/test_testsuite.py	2012-12-18 09:13:25 +0000
@@ -9,10 +9,11 @@
 from testtools import (
     ConcurrentTestSuite,
     iterate_tests,
+    PlaceHolder,
     TestCase,
     )
 from testtools.helpers import try_import
-from testtools.testsuite import FixtureSuite
+from testtools.testsuite import FixtureSuite, iterate_tests, sorted_tests
 from testtools.tests.helpers import LoggingResult
 
 FunctionFixture = try_import('fixtures.FunctionFixture')
@@ -93,6 +94,35 @@
         self.assertEqual(['setUp', 1, 2, 'tearDown'], log)
 
 
+class TestSortedTests(TestCase):
+
+    def test_sorts_custom_suites(self):
+        a = PlaceHolder('a')
+        b = PlaceHolder('b')
+        class Subclass(unittest.TestSuite):
+            def sort_tests(self):
+                self._tests = sorted_tests(self, True)
+        input_suite = Subclass([b, a])
+        suite = sorted_tests(input_suite)
+        self.assertEqual([a, b], list(iterate_tests(suite)))
+        self.assertEqual([input_suite], list(iter(suite)))
+
+    def test_custom_suite_without_sort_tests_works(self):
+        a = PlaceHolder('a')
+        b = PlaceHolder('b')
+        class Subclass(unittest.TestSuite):pass
+        input_suite = Subclass([b, a])
+        suite = sorted_tests(input_suite)
+        self.assertEqual([b, a], list(iterate_tests(suite)))
+        self.assertEqual([input_suite], list(iter(suite)))
+
+    def test_sorts_simple_suites(self):
+        a = PlaceHolder('a')
+        b = PlaceHolder('b')
+        suite = sorted_tests(unittest.TestSuite([b, a]))
+        self.assertEqual([a, b], list(iterate_tests(suite)))
+
+
 def test_suite():
     from unittest import TestLoader
     return TestLoader().loadTestsFromName(__name__)

=== modified file 'testtools/testsuite.py'
--- testtools/testsuite.py	2012-04-17 14:25:20 +0000
+++ testtools/testsuite.py	2012-12-18 09:13:25 +0000
@@ -6,9 +6,10 @@
 __all__ = [
   'ConcurrentTestSuite',
   'iterate_tests',
+  'sorted_tests',
   ]
 
-from testtools.helpers import try_imports
+from testtools.helpers import safe_hasattr, try_imports
 
 Queue = try_imports(['Queue.Queue', 'queue.Queue'])
 
@@ -114,3 +115,40 @@
             super(FixtureSuite, self).run(result)
         finally:
             self._fixture.cleanUp()
+
+    def sort_tests(self):
+        self._tests = sorted_tests(self, True)
+
+
+def _flatten_tests(suite_or_case, unpack_outer=False):
+    try:
+        tests = iter(suite_or_case)
+    except TypeError:
+        # Not iterable, assume its a test case.
+        return [(suite_or_case.id(), suite_or_case)]
+    if (type(suite_or_case) in (unittest.TestSuite,) or
+        unpack_outer):
+        # Plain old test suite (or any others we may add).
+        result = []
+        for test in tests:
+            # Recurse to flatten.
+            result.extend(_flatten_tests(test))
+        return result
+    else:
+        # Find any old actual test and grab its id.
+        suite_id = None
+        tests = iterate_tests(suite_or_case)
+        for test in tests:
+            suite_id = test.id()
+            break
+        # If it has a sort_tests method, call that.
+        if safe_hasattr(suite_or_case, 'sort_tests'):
+            suite_or_case.sort_tests()
+        return [(suite_id, suite_or_case)]
+
+
+def sorted_tests(suite_or_case, unpack_outer=False):
+    """Sort suite_or_case while preserving non-vanilla TestSuites."""
+    tests = _flatten_tests(suite_or_case, unpack_outer=unpack_outer)
+    tests.sort()
+    return unittest.TestSuite([test for (sort_key, test) in tests])