launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #24077
[Merge] ~cjwatson/launchpad:modern-zope-testrunner into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:modern-zope-testrunner into launchpad:master.
Commit message:
Upgrade to modern zope.testing and zope.testrunner
Launchpad has had a zope.testing fork since 2010, last changed in 2012,
and never merged from upstream. In the intervening period the relevant
code has been moved to zope.testrunner. I've landed most of our
substantial changes upstream. There are two remaining changes, now in a
much more manageable form:
* Report tests by id rather than str(test)
* Make --subunit --list-tests only print test IDs
This gets us quite a few useful improvements:
* Python 3 support in zope.testing and zope.testrunner
* subunit v2 protocol support (although using this will require
buildbot changes)
* Support for unexpected successes
* Improved support for skips
* Enable DeprecationWarning when running tests
The warning-related changes required some rearrangements to the way we
set up our own warning filters, since that now has to be done within
zope.testrunner's warnings.catch_warnings context manager.
I also included a few preparatory changes, mainly to tidy up handling of
stdout and stderr streams.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/374387
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:modern-zope-testrunner into launchpad:master.
diff --git a/constraints.txt b/constraints.txt
index fa23f05..33861d8 100644
--- a/constraints.txt
+++ b/constraints.txt
@@ -76,34 +76,10 @@ zope.structuredtext==3.5.1
zope.tal==4.3.0
zope.tales==3.5.3
#zope.testing==3.10.3
-# p1 Build of lp:~mars/zope.testing/3.9.4-p1. Fixes bugs 570380 and 587886.
-# p2 With patch for thread leaks to make them skips, fixes windmill errors
-# with 'new threads' in hudson/ec2 builds.
-# p3 And always tear down layers, because thats the Right Thing To Do.
-# p4 fixes --subunit --list to really just list the tests.
-# p5 Build of lp:~launchpad/zope.testing/3.9.4-p5. Fixes bug #609986.
-# p6 reinstates fix from p4. Build of lp:~launchpad/zope.testing/3.9.4-fork
-# revision 26.
-# p7 was unused
-# p8 redirects stdout and stderr to a black hole device when --subunit is used
-# p9 adds the redirection of __stderr__ to a black hole device
-# p10 changed the test reporting to use test.id() rather than
-# str(test) since only the id is unique.
-# p11 reverts p9.
-# p12 reverts p11, restoring p9.
-# p13 Add a new --require-unique flag to the testrunner. When set,
-# this will cause the testrunner to check all tests IDs to ensure they
-# haven't been loaded before. If it encounters a duplicate, it will
-# raise an error and quit.
-# p14 Adds test data written to stderr and stdout into the subunit output.
-# p15 Fixed internal tests.
-# p16 Adds support for skips in Python 2.7.
-# p17 Fixes skip support for Python 2.6.
-# To build (use Python 2.6) run "python bootstrap.py; ./bin/buildout". Then to
-# build the distribution run "bin/buildout setup . sdist"
-# Make sure you have subunit installed.
-zope.testing==3.9.4-p17
-zope.testrunner==4.0.4
+zope.testing==4.7
+#zope.testrunner==4.0.4
+# lp:~launchpad-committers/zope.testrunner:launchpad
+zope.testrunner[subunit]==5.1+lp1
zope.traversing==3.14.0
zope.viewlet==3.7.2
@@ -125,7 +101,7 @@ Pygments==2.2.0
#python-gettext==1.0
python-gettext==3.0
#python-subunit==0.0.7
-python-subunit==0.0.8beta
+python-subunit==1.3.0
#pytz==2014.10
pytz==2017.2
RestrictedPython==3.6.0
diff --git a/lib/lp/scripts/utilities/test.py b/lib/lp/scripts/utilities/test.py
index ee273cf..1899105 100755
--- a/lib/lp/scripts/utilities/test.py
+++ b/lib/lp/scripts/utilities/test.py
@@ -13,6 +13,7 @@
##############################################################################
"""Test script."""
+import argparse
import doctest
import os
import random
@@ -23,8 +24,9 @@ import time
import warnings
import six
-from zope.testing import testrunner
-from zope.testing.testrunner import options
+from zope.testrunner import options
+from zope.testrunner.feature import Feature
+from zope.testrunner.runner import Runner
from lp.scripts.utilities import (
importpedant,
@@ -63,9 +65,6 @@ def configure_environment():
# Install the import pedant import hook and atexit handler.
importpedant.install_import_pedant()
- # Install the warning handler hook and atexit handler.
- warninghandler.install_warning_handler()
-
# Ensure that atexit handlers are executed on TERM.
def exit_with_atexit_handlers(*ignored):
sys.exit(-1 * signal.SIGTERM)
@@ -103,9 +102,6 @@ def filter_warnings():
'ignore', 'twisted.python.plugin', DeprecationWarning,
)
warnings.filterwarnings(
- 'ignore', 'zope.testing.doctest', DeprecationWarning,
- )
- warnings.filterwarnings(
'ignore', 'bzrlib.*was deprecated', DeprecationWarning,
)
# The next one is caused by an infelicity in python-openid. The
@@ -126,6 +122,15 @@ def filter_warnings():
r'Use \"message.getOpenIDNamespace\(\)\" instead'),
DeprecationWarning
)
+ # XXX cjwatson 2019-10-18: This can be dropped once the port to Breezy
+ # is complete.
+ warnings.filterwarnings(
+ 'ignore',
+ re.escape(
+ 'RefsContainer._follow is deprecated. '
+ 'Use RefsContainer.follow instead.'),
+ DeprecationWarning, r'dulwich\.refs',
+ )
# This warning will be triggered if the beforeTraversal hook fails. We
# want to ensure it is not raised as an error, as this will mask the
# real problem.
@@ -141,6 +146,19 @@ def filter_warnings():
warnings.filterwarnings('error', r'shortlist\(\)')
+class LaunchpadWarnings(Feature):
+ """Install Launchpad's warning handler and filters."""
+
+ active = True
+
+ def global_setup(self):
+ warnings.showwarning = warninghandler.launchpad_showwarning
+ filter_warnings()
+
+ def global_teardown(self):
+ warninghandler.report_warnings()
+
+
def install_fake_pgsql_connect():
from lp.testing import pgsql
# If this is removed, make sure lp.testing.pgsql is updated
@@ -177,6 +195,13 @@ defaults = {
}
+class LaunchpadRunner(Runner):
+
+ def configure(self):
+ super(LaunchpadRunner, self).configure()
+ self.features.insert(0, LaunchpadWarnings(self))
+
+
def main():
# The working directory change is just so that the test script
# can be invoked from places other than the root of the source
@@ -187,7 +212,6 @@ def main():
fix_doctest_output()
configure_environment()
- filter_warnings()
install_fake_pgsql_connect()
randomise_listdir()
@@ -223,43 +247,46 @@ def main():
from lp.services.testing.parallel import main
sys.exit(main(sys.argv))
- def load_list(option, opt_str, list_name, parser):
- patch_find_tests(filter_tests(list_name, '--shuffle' in sys.argv))
- options.parser.add_option(
- '--load-list', type=str, action='callback', callback=load_list)
- options.parser.add_option(
+ class LoadListAction(argparse.Action):
+ def __call__(self, parser, namespace, values, option_string=None):
+ patch_find_tests(filter_tests(values, '--shuffle' in sys.argv))
+
+ options.parser.add_argument(
+ '--load-list', type=str, action=LoadListAction)
+ options.parser.add_argument(
'--parallel', action='store_true',
help='Run tests in parallel processes. '
'Poorly isolated tests will break.')
# tests_pattern is a regexp, so the parsed value is hard to compare
# with the default value in the loop below.
- options.parser.defaults['tests_pattern'] = defaults['tests_pattern']
+ options.parser.set_defaults(tests_pattern=defaults['tests_pattern'])
local_options = options.get_options(args=args)
# Set our default options, if the options aren't specified.
for name, value in defaults.items():
parsed_option = getattr(local_options, name)
- if ((parsed_option == []) or
- (parsed_option == options.parser.defaults.get(name))):
+ if (parsed_option == [] or
+ parsed_option == options.parser.get_default(name)):
# The option probably wasn't specified on the command line,
# let's replace it with our default value. It could be that
- # the real default (as specified in
- # zope.testing.testrunner.options) was specified, and we
- # shouldn't replace it with our default, but it's such and
- # edge case, so we don't have to care about it.
- options.parser.defaults[name] = value
+ # the real default (as specified in zope.testrunner.options)
+ # was specified, and we shouldn't replace it with our
+ # default, but it's such an edge case, so we don't have to
+ # care about it.
+ options.parser.set_defaults(**{name: value})
# Turn on Layer profiling if requested.
if local_options.verbose >= 3 and main_process:
profiled.setup_profiling()
try:
- try:
- testrunner.run([])
- except SystemExit:
- # Print Layer profiling report if requested.
- if main_process and local_options.verbose >= 3:
- profiled.report_profile_stats()
- raise
+ script_parts = [os.path.abspath(sys.argv[0])]
+ runner = LaunchpadRunner(
+ [], script_parts=script_parts, cwd=os.getcwd())
+ runner.run()
+ # Print Layer profiling report if requested.
+ if main_process and local_options.verbose >= 3:
+ profiled.report_profile_stats()
+ return int(runner.failed)
finally:
os.chdir(there)
diff --git a/lib/lp/scripts/utilities/warninghandler.py b/lib/lp/scripts/utilities/warninghandler.py
index addc5f3..4a489d7 100644
--- a/lib/lp/scripts/utilities/warninghandler.py
+++ b/lib/lp/scripts/utilities/warninghandler.py
@@ -1,11 +1,12 @@
-# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Handlers for warnings, to be installed when testing."""
+from __future__ import print_function
+
__metaclass__ = type
-import atexit
import inspect
import StringIO
import sys
@@ -188,39 +189,34 @@ def launchpad_showwarning(message, category, filename, lineno, file=None,
def report_need_page_titles():
global need_page_titles
if need_page_titles:
- print
- print "The following pages need titles."
+ print(file=sys.stderr)
+ print("The following pages need titles.", file=sys.stderr)
for message in need_page_titles:
- print " ", message
+ print(" ", message, file=sys.stderr)
def report_no_order_by():
global no_order_by
if no_order_by:
- print
- print ("The following code has issues with"
- " ambiguous select results ordering.")
+ print(file=sys.stderr)
+ print("The following code has issues with"
+ " ambiguous select results ordering.", file=sys.stderr)
for report in no_order_by:
- print
- print report
+ print(file=sys.stderr)
+ print(report, file=sys.stderr)
def report_other_warnings():
global other_warnings
if other_warnings:
- print
- print "General warnings."
+ print(file=sys.stderr)
+ print("General warnings.", file=sys.stderr)
for warninginfo in other_warnings.itervalues():
- print
- print warninginfo
+ print(file=sys.stderr)
+ print(warninginfo, file=sys.stderr)
def report_warnings():
report_need_page_titles()
report_no_order_by()
report_other_warnings()
-
-
-def install_warning_handler():
- warnings.showwarning = launchpad_showwarning
- atexit.register(report_warnings)
diff --git a/lib/lp/services/mail/tests/test_stub.py b/lib/lp/services/mail/tests/test_stub.py
index 095af90..8945153 100644
--- a/lib/lp/services/mail/tests/test_stub.py
+++ b/lib/lp/services/mail/tests/test_stub.py
@@ -1,4 +1,4 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
__metaclass__ = type
@@ -6,7 +6,7 @@ __metaclass__ = type
from doctest import DocTestSuite
import re
-from zope.testing.renormalizing import RENormalizing
+from zope.testing.renormalizing import OutputChecker
from lp.testing.layers import LaunchpadFunctionalLayer
@@ -112,7 +112,7 @@ def test_simple_sendmail():
def test_suite():
- suite = DocTestSuite(checker=RENormalizing([
+ suite = DocTestSuite(checker=OutputChecker([
(re.compile(r"'revision', '[0-9a-f]+'"),
"'revision', '%s'" % ('0' * 40))]))
suite.layer = LaunchpadFunctionalLayer
diff --git a/lib/lp/services/testing/customresult.py b/lib/lp/services/testing/customresult.py
index 5f1bbc4..2164f3d 100644
--- a/lib/lp/services/testing/customresult.py
+++ b/lib/lp/services/testing/customresult.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2012 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Support code for using a custom test result in test.py."""
@@ -11,11 +11,11 @@ __all__ = [
from unittest import TestSuite
-from zope.testing.testrunner import find
+from zope.testrunner import find
def patch_find_tests(hook):
- """Add a post-processing hook to zope.testing.testrunner.find_tests.
+ """Add a post-processing hook to zope.testrunner.find_tests.
This is useful for things like filtering tests or listing tests.
diff --git a/lib/lp/services/testing/tests/test_parallel.py b/lib/lp/services/testing/tests/test_parallel.py
index 9e65111..2dec513 100644
--- a/lib/lp/services/testing/tests/test_parallel.py
+++ b/lib/lp/services/testing/tests/test_parallel.py
@@ -1,4 +1,4 @@
-# Copyright 2010 Canonical Ltd. This software is licensed under the
+# Copyright 2010-2019 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Parallel test glue."""
@@ -102,7 +102,7 @@ class TestUtilities(TestCase, TestWithFixtures):
self.assertEqual(
['bin/test', '-vt', 'filter', '--list-tests', '--subunit'],
args['args'])
- return {'stdin': StringIO(), 'stdout': StringIO(u"""\
+ return {'stdin': StringIO(), 'stdout': StringIO("""\
test: quux
successful: quux
test: glom
diff --git a/lib/lp/services/tests/test_stacktrace.py b/lib/lp/services/tests/test_stacktrace.py
index 0f1c605..dedbf77 100644
--- a/lib/lp/services/tests/test_stacktrace.py
+++ b/lib/lp/services/tests/test_stacktrace.py
@@ -1,20 +1,22 @@
-# Copyright 2011 Canonical Ltd. This software is licensed under the
+# Copyright 2011-2019 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Test the stacktrace module."""
__metaclass__ = type
-import StringIO
import sys
+from fixtures import MonkeyPatch
+import six
+
from lp.services import stacktrace
from lp.testing import TestCase
from lp.testing.layers import BaseLayer
# This constant must always be equal to the line number on which it lives for
# the tests to pass.
-MY_LINE_NUMBER = 17
+MY_LINE_NUMBER = 19
MY_FILE_NAME = __file__[:__file__.rindex('.py')] + '.py'
@@ -148,17 +150,16 @@ class TestStacktrace(TestCase):
def test_get_frame_data_supplement_bad_getInfo_with_traceback(self):
def boo_hiss():
raise ValueError()
- original_stderr = sys.__stderr__
- stderr = sys.stderr = StringIO.StringIO()
+ stderr = six.StringIO()
self.assertFalse(stacktrace.DEBUG_EXCEPTION_FORMATTER)
stacktrace.DEBUG_EXCEPTION_FORMATTER = True
try:
- filename, lineno, name, line, modname, supplement, info = (
- stacktrace._get_frame_data(
- get_frame(supplement=dict(getInfo=boo_hiss)),
- MY_LINE_NUMBER))
+ with MonkeyPatch('sys.stderr', stderr):
+ filename, lineno, name, line, modname, supplement, info = (
+ stacktrace._get_frame_data(
+ get_frame(supplement=dict(getInfo=boo_hiss)),
+ MY_LINE_NUMBER))
finally:
- sys.stderr = original_stderr
stacktrace.DEBUG_EXCEPTION_FORMATTER = False
self.assertEqual(
dict(source_url=None, line=None, column=None, expression=None,
@@ -276,50 +277,43 @@ class TestStacktrace(TestCase):
def test_format_list_extra_errors(self):
extracted = stacktrace.extract_stack(get_frame(supplement=dict()))
extracted[-1][-2]['warnings'] = object() # This should never happen.
- original_stderr = sys.__stderr__
- stderr = sys.stderr = StringIO.StringIO()
+ stderr = six.StringIO()
self.assertFalse(stacktrace.DEBUG_EXCEPTION_FORMATTER)
stacktrace.DEBUG_EXCEPTION_FORMATTER = True
try:
- formatted = stacktrace.format_list(extracted)
+ with MonkeyPatch('sys.stderr', stderr):
+ formatted = stacktrace.format_list(extracted)
finally:
- sys.stderr = original_stderr
stacktrace.DEBUG_EXCEPTION_FORMATTER = False
self.assertStartsWith(stderr.getvalue(), 'Traceback (most recent')
self.assertEndsWith(formatted[-1], ' return sys._getframe()\n')
def test_print_list_default(self):
extracted = stacktrace.extract_stack(get_frame())
- original_stderr = sys.__stderr__
- stderr = sys.stderr = StringIO.StringIO()
- try:
+ stderr = six.StringIO()
+ with MonkeyPatch('sys.stderr', stderr):
stacktrace.print_list(extracted)
- finally:
- sys.stderr = original_stderr
self.assertEndsWith(stderr.getvalue(), 'return sys._getframe()\n')
def test_print_list_file(self):
extracted = stacktrace.extract_stack(get_frame())
- f = StringIO.StringIO()
+ f = six.StringIO()
stacktrace.print_list(extracted, file=f)
self.assertEndsWith(f.getvalue(), 'return sys._getframe()\n')
def test_print_stack_default(self):
- original_stderr = sys.__stderr__
- stderr = sys.stderr = StringIO.StringIO()
- try:
+ stderr = six.StringIO()
+ with MonkeyPatch('sys.stderr', stderr):
stacktrace.print_stack()
- finally:
- sys.stderr = original_stderr
self.assertEndsWith(stderr.getvalue(), 'stacktrace.print_stack()\n')
def test_print_stack_options(self):
- f = StringIO.StringIO()
+ f = six.StringIO()
frame = get_frame()
stacktrace.print_stack(f=frame, limit=100, file=f)
self.assertEndsWith(f.getvalue(), 'return sys._getframe()\n')
self.assertTrue(len(f.getvalue().split('\n')) > 4)
- f = StringIO.StringIO()
+ f = six.StringIO()
stacktrace.print_stack(f=frame, limit=2, file=f)
self.assertEqual(4, len(f.getvalue().strip().split('\n')))
self.assertEndsWith(f.getvalue(), 'return sys._getframe()\n')
diff --git a/lib/lp/testing/__init__.py b/lib/lp/testing/__init__.py
index 5f91c86..1aa11ea 100644
--- a/lib/lp/testing/__init__.py
+++ b/lib/lp/testing/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
from __future__ import absolute_import, print_function
@@ -124,7 +124,7 @@ from zope.security.proxy import (
isinstance as zope_isinstance,
removeSecurityProxy,
)
-from zope.testing.testrunner.runner import TestResult as ZopeTestResult
+from zope.testrunner.runner import TestResult as ZopeTestResult
from lp.app.interfaces.launchpad import ILaunchpadCelebrities
from lp.app.interfaces.security import IAuthorization
@@ -243,8 +243,8 @@ def reset_logging():
logging._handlers.clear()
# Reset the setup
- from zope.testing.testrunner.runner import Runner
- from zope.testing.testrunner.logsupport import Logging
+ from zope.testrunner.runner import Runner
+ from zope.testrunner.logsupport import Logging
Logging(Runner()).global_setup()
lp_sitecustomize.customize_logger()
diff --git a/setup.py b/setup.py
index 6859381..a2141cf 100644
--- a/setup.py
+++ b/setup.py
@@ -272,6 +272,7 @@ setup(
'zope.tales',
'zope.testbrowser',
'zope.testing',
+ 'zope.testrunner[subunit]',
'zope.traversing',
'zope.viewlet', # only fixing a broken dependency
'zope.vocabularyregistry',
