testtools-dev team mailing list archive
-
testtools-dev team
-
Mailing list archive
-
Message #00322
[Merge] lp:~jml/testtools/more-doc-improvements into lp:testtools
Jonathan Lange has proposed merging lp:~jml/testtools/more-doc-improvements into lp:testtools.
Requested reviews:
testtools developers (testtools-dev)
Related bugs:
#661116 MultiTestResult not documented
https://bugs.launchpad.net/bugs/661116
This branch:
* moves some of the main documentation into a new 'doc' directory
* adds a sphinx-based doc building system
* completely revises the manual, breaking it up into two parts: for test authors and for framework folk
* updates, tweaks and polishes the HACKING guide.
An example build of the docs can be found at http://mumak.net/testtools/docs/.
--
https://code.launchpad.net/~jml/testtools/more-doc-improvements/+merge/42045
Your team testtools developers is requested to review the proposed merge of lp:~jml/testtools/more-doc-improvements into lp:testtools.
=== modified file '.bzrignore'
--- .bzrignore 2010-07-29 17:19:32 +0000
+++ .bzrignore 2010-11-28 15:19:56 +0000
@@ -6,3 +6,4 @@
TAGS
apidocs
_trial_temp
+doc/_build
=== modified file 'Makefile'
--- Makefile 2010-11-11 02:22:43 +0000
+++ Makefile 2010-11-28 15:19:56 +0000
@@ -12,7 +12,7 @@
tags: ${SOURCES}
ctags -R testtools/
-clean:
+clean: clean-sphinx
rm -f TAGS tags
find testtools -name "*.pyc" -exec rm '{}' \;
@@ -26,10 +26,24 @@
snapshot: prerelease
./setup.py sdist
+### Documentation ###
+
apidocs:
pydoctor --make-html --add-package testtools \
--docformat=restructuredtext --project-name=testtools \
--project-url=https://launchpad.net/testtools
-
-.PHONY: check clean prerelease release apidocs
+docs: docs-sphinx
+
+docs-sphinx: html-sphinx
+
+# Clean out generated documentation
+clean-sphinx:
+ cd doc && make clean
+
+# Build the html docs using Sphinx.
+html-sphinx:
+ cd doc && make html
+
+.PHONY: apidocs docs-sphinx clean-sphinx html-sphinx docs
+.PHONY: check clean prerelease release
=== added directory 'doc'
=== added file 'doc/Makefile'
--- doc/Makefile 1970-01-01 00:00:00 +0000
+++ doc/Makefile 2010-11-28 15:19:56 +0000
@@ -0,0 +1,89 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS =
+SPHINXBUILD = sphinx-build
+PAPER =
+BUILDDIR = _build
+
+# Internal variables.
+PAPEROPT_a4 = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest
+
+help:
+ @echo "Please use \`make <target>' where <target> is one of"
+ @echo " html to make standalone HTML files"
+ @echo " dirhtml to make HTML files named index.html in directories"
+ @echo " pickle to make pickle files"
+ @echo " json to make JSON files"
+ @echo " htmlhelp to make HTML files and a HTML help project"
+ @echo " qthelp to make HTML files and a qthelp project"
+ @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+ @echo " changes to make an overview of all changed/added/deprecated items"
+ @echo " linkcheck to check all external links for integrity"
+ @echo " doctest to run all doctests embedded in the documentation (if enabled)"
+
+clean:
+ -rm -rf $(BUILDDIR)/*
+
+html:
+ $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+ @echo
+ @echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+dirhtml:
+ $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+ @echo
+ @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+pickle:
+ $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+ @echo
+ @echo "Build finished; now you can process the pickle files."
+
+json:
+ $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+ @echo
+ @echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+ $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+ @echo
+ @echo "Build finished; now you can run HTML Help Workshop with the" \
+ ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+ $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+ @echo
+ @echo "Build finished; now you can run "qcollectiongenerator" with the" \
+ ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+ @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/testtools.qhcp"
+ @echo "To view the help file:"
+ @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/testtools.qhc"
+
+latex:
+ $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+ @echo
+ @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+ @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \
+ "run these through (pdf)latex."
+
+changes:
+ $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+ @echo
+ @echo "The overview file is in $(BUILDDIR)/changes."
+
+linkcheck:
+ $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+ @echo
+ @echo "Link check complete; look for any errors in the above output " \
+ "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+ $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+ @echo "Testing of doctests in the sources finished, look at the " \
+ "results in $(BUILDDIR)/doctest/output.txt."
=== added directory 'doc/_static'
=== added directory 'doc/_templates'
=== added file 'doc/conf.py'
--- doc/conf.py 1970-01-01 00:00:00 +0000
+++ doc/conf.py 2010-11-28 15:19:56 +0000
@@ -0,0 +1,194 @@
+# -*- coding: utf-8 -*-
+#
+# testtools documentation build configuration file, created by
+# sphinx-quickstart on Sun Nov 28 13:45:40 2010.
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys, os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#sys.path.append(os.path.abspath('.'))
+
+# -- General configuration -----------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = ['sphinx.ext.autodoc']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'testtools'
+copyright = u'2010, The testtools authors'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = 'VERSION'
+# The full version, including alpha/beta/rc tags.
+release = 'VERSION'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of documents that shouldn't be included in the build.
+#unused_docs = []
+
+# List of directories, relative to source directory, that shouldn't be searched
+# for source files.
+exclude_trees = ['_build']
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+
+# -- Options for HTML output ---------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. Major themes that come with
+# Sphinx are currently 'default' and 'sphinxdoc'.
+html_theme = 'default'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further. For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents. If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar. Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_use_modindex = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it. The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = ''
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'testtoolsdoc'
+
+
+# -- Options for LaTeX output --------------------------------------------------
+
+# The paper size ('letter' or 'a4').
+#latex_paper_size = 'letter'
+
+# The font size ('10pt', '11pt' or '12pt').
+#latex_font_size = '10pt'
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, documentclass [howto/manual]).
+latex_documents = [
+ ('index', 'testtools.tex', u'testtools Documentation',
+ u'The testtools authors', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# Additional stuff for the LaTeX preamble.
+#latex_preamble = ''
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_use_modindex = True
=== renamed file 'MANUAL' => 'doc/for-framework-folk.rst'
--- MANUAL 2010-11-25 11:32:24 +0000
+++ doc/for-framework-folk.rst 2010-11-28 15:19:56 +0000
@@ -1,51 +1,66 @@
-======
-Manual
-======
+============================
+testtools for framework folk
+============================
Introduction
-------------
-
-This document provides overview of the features provided by testtools. Refer
-to the API docs (i.e. docstrings) for full details on a particular feature.
+============
+
+In addition to having many features :doc:`for test authors
+<for-test-authors>`, testtools also has many bits and pieces that are useful
+for folk who writing testing frameworks.
+
+If you are the author of a test runner, are working on a very large
+unit-tested project, are trying to get one testing framework to play nicely
+with another or are hacking away at getting your test suite to run in parallel
+over a heterogenous cluster of machines, this guide is for you.
+
+This manual is a summary. You can get details by consulting the `testtools
+API docs`_.
+
Extensions to TestCase
-----------------------
+======================
Custom exception handling
-~~~~~~~~~~~~~~~~~~~~~~~~~
+-------------------------
testtools provides a way to control how test exceptions are handled. To do
-this, add a new exception to self.exception_handlers on a TestCase. For
-example::
+this, add a new exception to ``self.exception_handlers`` on a
+``testtools.TestCase``. For example::
>>> self.exception_handlers.insert(-1, (ExceptionClass, handler)).
-Having done this, if any of setUp, tearDown, or the test method raise
-ExceptionClass, handler will be called with the test case, test result and the
-raised exception.
+Having done this, if any of ``setUp``, ``tearDown``, or the test method raise
+``ExceptionClass``, ``handler`` will be called with the test case, test result
+and the raised exception.
+
+Use this if you want to add a new kind of test result, that is, if you think
+that ``addError``, ``addFailure`` and so forth are not enough for your needs.
+
Controlling test execution
-~~~~~~~~~~~~~~~~~~~~~~~~~~
+--------------------------
If you want to control more than just how exceptions are raised, you can
-provide a custom `RunTest` to a TestCase. The `RunTest` object can change
-everything about how the test executes.
-
-To work with `testtools.TestCase`, a `RunTest` must have a factory that takes
-a test and an optional list of exception handlers. Instances returned by the
-factory must have a `run()` method that takes an optional `TestResult` object.
-
-The default is `testtools.runtest.RunTest` and calls 'setUp', the test method
-and 'tearDown' in the normal, vanilla way that Python's standard unittest
-does.
-
-To specify a `RunTest` for all the tests in a `TestCase` class, do something
+provide a custom ``RunTest`` to a ``TestCase``. The ``RunTest`` object can
+change everything about how the test executes.
+
+To work with ``testtools.TestCase``, a ``RunTest`` must have a factory that
+takes a test and an optional list of exception handlers. Instances returned
+by the factory must have a ``run()`` method that takes an optional ``TestResult``
+object.
+
+The default is ``testtools.runtest.RunTest``, which calls ``setUp``, the test
+method, ``tearDown`` and clean ups (see :ref:`addCleanup`) in the normal, vanilla
+way that Python's standard unittest_ does.
+
+To specify a ``RunTest`` for all the tests in a ``TestCase`` class, do something
like this::
class SomeTests(TestCase):
run_tests_with = CustomRunTestFactory
-To specify a `RunTest` for a specific test in a `TestCase` class, do::
+To specify a ``RunTest`` for a specific test in a ``TestCase`` class, do::
class SomeTests(TestCase):
@run_test_with(CustomRunTestFactory, extra_arg=42, foo='whatever')
@@ -53,150 +68,11 @@
pass
In addition, either of these can be overridden by passing a factory in to the
-`TestCase` constructor with the optional 'runTest' argument.
-
-TestCase.addCleanup
-~~~~~~~~~~~~~~~~~~~
-
-addCleanup is a robust way to arrange for a cleanup function to be called
-before tearDown. This is a powerful and simple alternative to putting cleanup
-logic in a try/finally block or tearDown method. e.g.::
-
- def test_foo(self):
- foo.lock()
- self.addCleanup(foo.unlock)
- ...
-
-Cleanups can also report multiple errors, if appropriate by wrapping them in
-a testtools.MultipleExceptions object::
-
- raise MultipleExceptions(exc_info1, exc_info2)
-
-
-TestCase.addOnException
-~~~~~~~~~~~~~~~~~~~~~~~
-
-addOnException adds an exception handler that will be called from the test
-framework when it detects an exception from your test code. The handler is
-given the exc_info for the exception, and can use this opportunity to attach
-more data (via the addDetails API) and potentially other uses.
-
-
-TestCase.patch
-~~~~~~~~~~~~~~
-
-``patch`` is a convenient way to monkey-patch a Python object for the duration
-of your test. It's especially useful for testing legacy code. e.g.::
-
- def test_foo(self):
- my_stream = StringIO()
- self.patch(sys, 'stderr', my_stream)
- run_some_code_that_prints_to_stderr()
- self.assertEqual('', my_stream.getvalue())
-
-The call to ``patch`` above masks sys.stderr with 'my_stream' so that anything
-printed to stderr will be captured in a StringIO variable that can be actually
-tested. Once the test is done, the real sys.stderr is restored to its rightful
-place.
-
-
-TestCase.skipTest
-~~~~~~~~~~~~~~~~~
-
-``skipTest`` is a simple way to have a test stop running and be reported as a
-skipped test, rather than a success/error/failure. This is an alternative to
-convoluted logic during test loading, permitting later and more localized
-decisions about the appropriateness of running a test. Many reasons exist to
-skip a test - for instance when a dependency is missing, or if the test is
-expensive and should not be run while on laptop battery power, or if the test
-is testing an incomplete feature (this is sometimes called a TODO). Using this
-feature when running your test suite with a TestResult object that is missing
-the ``addSkip`` method will result in the ``addError`` method being invoked
-instead. ``skipTest`` was previously known as ``skip`` but as Python 2.7 adds
-``skipTest`` support, the ``skip`` name is now deprecated (but no warning
-is emitted yet - some time in the future we may do so).
-
-TestCase.useFixture
-~~~~~~~~~~~~~~~~~~~
-
-``useFixture(fixture)`` calls setUp on the fixture, schedules a cleanup to
-clean it up, and schedules a cleanup to attach all details held by the
-fixture to the details dict of the test case. The fixture object should meet
-the ``fixtures.Fixture`` protocol (version 0.3.4 or newer). This is useful
-for moving code out of setUp and tearDown methods and into composable side
-classes.
-
-
-New assertion methods
-~~~~~~~~~~~~~~~~~~~~~
-
-testtools adds several assertion methods:
-
- * assertIn
- * assertNotIn
- * assertIs
- * assertIsNot
- * assertIsInstance
- * assertThat
-
-
-Improved assertRaises
-~~~~~~~~~~~~~~~~~~~~~
-
-TestCase.assertRaises returns the caught exception. This is useful for
-asserting more things about the exception than just the type::
-
- error = self.assertRaises(UnauthorisedError, thing.frobnicate)
- self.assertEqual('bob', error.username)
- self.assertEqual('User bob cannot frobnicate', str(error))
-
-Note that this is incompatible with the assertRaises in unittest2/Python2.7.
-While we have no immediate plans to change to be compatible consider using the
-new assertThat facility instead::
-
- self.assertThat(
- lambda: thing.frobnicate('foo', 'bar'),
- Raises(MatchesException(UnauthorisedError('bob')))
-
-There is also a convenience function to handle this common case::
-
- self.assertThat(
- lambda: thing.frobnicate('foo', 'bar'),
- raises(UnauthorisedError('bob')))
-
-
-TestCase.assertThat
-~~~~~~~~~~~~~~~~~~~
-
-assertThat is a clean way to write complex assertions without tying them to
-the TestCase inheritance hierarchy (and thus making them easier to reuse).
-
-assertThat takes an object to be matched, and a matcher, and fails if the
-matcher does not match the matchee.
-
-See pydoc testtools.Matcher for the protocol that matchers need to implement.
-
-testtools includes some matchers in testtools.matchers.
-python -c 'import testtools.matchers; print testtools.matchers.__all__' will
-list those matchers.
-
-An example using the DocTestMatches matcher which uses doctests example
-matching logic::
-
- def test_foo(self):
- self.assertThat([1,2,3,4], DocTestMatches('[1, 2, 3, 4]'))
-
-
-Creation methods
-~~~~~~~~~~~~~~~~
-
-testtools.TestCase implements creation methods called ``getUniqueString`` and
-``getUniqueInteger``. See pages 419-423 of *xUnit Test Patterns* by Meszaros
-for a detailed discussion of creation methods.
+``TestCase`` constructor with the optional ``runTest`` argument.
Test renaming
-~~~~~~~~~~~~~
+-------------
``testtools.clone_test_with_new_id`` is a function to copy a test case
instance to one with a new name. This is helpful for implementing test
@@ -204,10 +80,10 @@
Extensions to TestResult
-------------------------
+========================
TestResult.addSkip
-~~~~~~~~~~~~~~~~~~
+------------------
This method is called on result objects when a test skips. The
``testtools.TestResult`` class records skips in its ``skip_reasons`` instance
@@ -215,17 +91,17 @@
TestResult.time
-~~~~~~~~~~~~~~~
+---------------
-This method controls the time used by a TestResult, permitting accurate
+This method controls the time used by a ``TestResult``, permitting accurate
timing of test results gathered on different machines or in different threads.
See pydoc testtools.TestResult.time for more details.
ThreadsafeForwardingResult
-~~~~~~~~~~~~~~~~~~~~~~~~~~
+--------------------------
-A TestResult which forwards activity to another test result, but synchronises
+A ``TestResult`` which forwards activity to another test result, but synchronises
on a semaphore to ensure that all the activity for a single test arrives in a
batch. This allows simple TestResults which do not expect concurrent test
reporting to be fed the activity from multiple test threads, or processes.
@@ -234,10 +110,26 @@
each error as a distinct complete test.
+MultiTestResult
+---------------
+
+A test result that dispatches its events to many test results. Use this
+to combine multiple different test result objects into one test result object
+that can be passed to ``TestCase.run()`` or similar. For example::
+
+ a = TestResult()
+ b = TestResult()
+ combined = MultiTestResult(a, b)
+ combined.startTestRun() # Calls a.startTestRun() and b.startTestRun()
+
+Each of the methods on ``MultiTestResult`` will return a tuple of whatever the
+component test results return.
+
+
TextTestResult
-~~~~~~~~~~~~~~
+--------------
-A TestResult that provides a text UI very similar to the Python standard
+A ``TestResult`` that provides a text UI very similar to the Python standard
library UI. Key differences are that its supports the extended outcomes and
details API, and is completely encapsulated into the result object, permitting
it to be used without a 'TestRunner' object. Not all the Python 2.7 outcomes
@@ -245,32 +137,40 @@
These limitations will be corrected soon.
+ExtendedToOriginalDecorator
+---------------------------
+
+Adapts legacy ``TestResult`` objects, such as those found in older Pythons, to
+meet the testtools ``TestResult`` API.
+
+
Test Doubles
-~~~~~~~~~~~~
+------------
In testtools.testresult.doubles there are three test doubles that testtools
-uses for its own testing: Python26TestResult, Python27TestResult,
-ExtendedTestResult. These TestResult objects implement a single variation of
-the TestResult API each, and log activity to a list self._events. These are
+uses for its own testing: ``Python26TestResult``, ``Python27TestResult``,
+``ExtendedTestResult``. These TestResult objects implement a single variation of
+the TestResult API each, and log activity to a list ``self._events``. These are
made available for the convenience of people writing their own extensions.
startTestRun and stopTestRun
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+----------------------------
-Python 2.7 added hooks 'startTestRun' and 'stopTestRun' which are called
+Python 2.7 added hooks ``startTestRun`` and ``stopTestRun`` which are called
before and after the entire test run. 'stopTestRun' is particularly useful for
test results that wish to produce summary output.
-testtools.TestResult provides empty startTestRun and stopTestRun methods, and
-the default testtools runner will call these methods appropriately.
+``testtools.TestResult`` provides empty ``startTestRun`` and ``stopTestRun``
+methods, and the default testtools runner will call these methods
+appropriately.
Extensions to TestSuite
------------------------
+=======================
ConcurrentTestSuite
-~~~~~~~~~~~~~~~~~~~
+-------------------
A TestSuite for parallel testing. This is used in conjuction with a helper that
runs a single suite in some parallel fashion (for instance, forking, handing
@@ -279,71 +179,5 @@
objects with a run(result), runs them all in threads using the
ThreadsafeForwardingResult to coalesce their activity.
-
-Running tests
--------------
-
-testtools provides a convenient way to run a test suite using the testtools
-result object: python -m testtools.run testspec [testspec...].
-
-To run tests with Python 2.4, you'll have to do something like:
- python2.4 /path/to/testtools/run.py testspec [testspec ...].
-
-
-Test discovery
---------------
-
-testtools includes a backported version of the Python 2.7 glue for using the
-discover test discovery module. If you either have Python 2.7/3.1 or newer, or
-install the 'discover' module, then you can invoke discovery::
-
- python -m testtools.run discover [path]
-
-For more information see the Python 2.7 unittest documentation, or::
-
- python -m testtools.run --help
-
-
-Twisted support
----------------
-
-Support for running Twisted tests is very experimental right now. You
-shouldn't really do it. However, if you are going to, here are some tips for
-converting your Trial tests into testtools tests.
-
- * Use the AsynchronousDeferredRunTest runner
- * Make sure to upcall to setUp and tearDown
- * Don't use setUpClass or tearDownClass
- * Don't expect setting .todo, .timeout or .skip attributes to do anything
- * flushLoggedErrors is not there for you. Sorry.
- * assertFailure is not there for you. Even more sorry.
-
-
-General helpers
----------------
-
-Lots of the time we would like to conditionally import modules. testtools
-needs to do this itself, and graciously extends the ability to its users.
-
-Instead of::
-
- try:
- from twisted.internet import defer
- except ImportError:
- defer = None
-
-You can do::
-
- defer = try_import('twisted.internet.defer')
-
-
-Instead of::
-
- try:
- from StringIO import StringIO
- except ImportError:
- from io import StringIO
-
-You can do::
-
- StringIO = try_imports(['StringIO.StringIO', 'io.StringIO'])
+.. _`testtools API docs`: http://mumak.net/testtools/apidocs/
+.. _unittest: http://docs.python.org/library/unittest.html
=== added file 'doc/for-test-authors.rst'
--- doc/for-test-authors.rst 1970-01-01 00:00:00 +0000
+++ doc/for-test-authors.rst 2010-11-28 15:19:56 +0000
@@ -0,0 +1,963 @@
+==========================
+testtools for test authors
+==========================
+
+If you are writing tests for a Python project and you (rather wisely) want to
+use testtools to do so, this is the manual for you.
+
+We assume that you already know Python and that you know something about
+automated testing already.
+
+If you are a test author of an unusually large or unusually unusual test
+suite, you might be interested in :doc:`for-framework-folk`.
+
+You might also be interested in the `testtools API docs`_.
+
+
+Introduction
+============
+
+testtools is a set of extensions to Python's standard unittest module.
+Writing tests with testtools is very much like writing tests with standard
+Python, or with Twisted's "trial_", or nose_, except a little bit easier and
+more enjoyable.
+
+Below, we'll try to give some examples of how to use testtools in its most
+basic way, as well as a sort of feature-by-feature breakdown of the cool bits
+that you could easily miss.
+
+
+The basics
+==========
+
+Here's what a basic testtools unit tests look like::
+
+ from testtools import TestCase
+ from myproject import silly
+
+ class TestSillySquare(TestCase):
+ """Tests for silly square function."""
+
+ def test_square(self):
+ # 'square' takes a number and multiplies it by itself.
+ result = silly.square(7)
+ self.assertEqual(result, 49)
+
+ def test_square_bad_input(self):
+ # 'square' raises a TypeError if it's given bad input, say a
+ # string.
+ self.assertRaises(TypeError, silly.square, "orange")
+
+
+Here you have a class that inherits from ``testtools.TestCase`` and bundles
+together a bunch of related tests. The tests themselves are methods on that
+class that begin with ``test_``.
+
+Running your tests
+------------------
+
+You can run these tests in many ways. testtools provides a very basic
+mechanism for doing so::
+
+ $ python -m testtools.run exampletest
+ Tests running...
+ Ran 2 tests in 0.000s
+
+ OK
+
+where 'exampletest' is a module that contains unit tests. By default,
+``testtools.run`` will *not* recursively search the module or package for unit
+tests. To do this, you will need to either have the discover_ module
+installed or have Python 2.7 or later, and then run::
+
+ $ python -m testtools.run discover packagecontainingtests
+
+For more information see the Python 2.7 unittest documentation, or::
+
+ python -m testtools.run --help
+
+As your testing needs grow and evolve, you will probably want to use a more
+sophisticated test runner. There are many of these for Python, and almost all
+of them will happily run testtools tests. In particular:
+
+* testrepository_
+* Trial_
+* nose_
+* unittest2_
+* `zope.testrunner`_ (aka zope.testing)
+
+From now on, we'll assume that you know how to run your tests.
+
+
+Assertions
+==========
+
+The core of automated testing is making assertions about the way things are,
+and getting a nice, helpful, informative error message when things are not as
+they ought to be.
+
+All of the assertions that you can find in Python standard unittest_ can be
+found in testtools (remember, testtools extends unittest). testtools changes
+the behaviour of some of those assertions slightly and adds some new
+assertions that you will almost certainly find useful.
+
+
+Improved assertRaises
+---------------------
+
+``TestCase.assertRaises`` returns the caught exception. This is useful for
+asserting more things about the exception than just the type::
+
+ def test_square_bad_input(self):
+ # 'square' raises a TypeError if it's given bad input, say a
+ # string.
+ e = self.assertRaises(TypeError, silly.square, "orange")
+ self.assertEqual("orange", e.bad_value)
+ self.assertEqual("Cannot square 'orange', not a number.", str(e))
+
+Note that this is incompatible with the ``assertRaises`` in unittest2 and
+Python2.7, and that we have no immediate plans it to be so.
+
+
+assertIn, assertNotIn
+---------------------
+
+These two assertions check whether a value is in a sequence and whether a
+value is not in a sequence. They are "assert" versions of the ``in`` and
+``not in`` operators. For example::
+
+ def test_assert_in_example(self):
+ self.assertIn('a', 'cat')
+ self.assertNotIn('o', 'cat')
+ self.assertIn(5, list_of_primes_under_ten)
+ self.assertNotIn(12, list_of_primes_under_ten)
+
+
+assertIs, assertIsNot
+---------------------
+
+These two assertions check whether values are identical to one another. This
+is sometimes useful when you want to test something more strict than mere
+equality. For example::
+
+ def test_assert_is_example(self):
+ foo = [None]
+ foo_alias = foo
+ bar = [None]
+ self.assertIs(foo, foo_alias)
+ self.assertIsNot(foo, bar)
+ self.assertEqual(foo, bar) # They are equal, but not identical
+
+
+assertIsInstance
+----------------
+
+As much as we love duck-typing and polymorphism, sometimes you need to check
+whether or not a value is of a given type. This method does that. For
+example::
+
+ def test_assert_is_instance_example(self):
+ now = datetime.now()
+ self.assertIsInstance(now, datetime)
+
+Note that there is no ``assertIsNotInstance`` in testtools currently.
+
+
+Matchers
+========
+
+The built-in assertion methods are very useful, they are the bread and butter
+of writing tests. However, soon enough you will probably want to write your
+own assertions. Perhaps there are domain specific things that you want to
+check (e.g. assert that two widgets are aligned parallel to the flux grid), or
+perhaps you want to check something that could almost but not quite be found
+in some other standard library (e.g. assert that two paths point to the same
+file).
+
+When you are in such situations, you could either make a base class for your
+project that inherits from ``testtools.TestCase`` and make sure that all of
+your tests derive from that, *or* you could use testtools ``Matcher`` system.
+
+
+Using Matchers
+--------------
+
+Here's a really basic example using stock matchers found in testtools::
+
+ import testtools
+ from testtools.matchers import Equals
+
+ class TestSquare(TestCase):
+ def test_square(self):
+ result = square(7)
+ self.assertThat(result, Equals(49))
+
+The line ``self.assertThat(result, Equals(49))`` is equivalent to
+``self.assertEqual(result, 49)`` and means "assert that ``result`` equals 49".
+The difference is that ``assertThat`` is a more general method that takes some
+kind of observed value (in this case, ``result``) and any matcher object
+(here, ``Equals(49)``).
+
+The matcher object could be absolutely anything that implements the Matcher
+protocol. This means that you can make more complex matchers by combining
+existing ones::
+
+ def test_square_silly(self):
+ result = square(7)
+ self.assertThat(result, Not(Equals(50)))
+
+Which is roughly equivalent to::
+
+ def test_square_silly(self):
+ result = square(7)
+ self.assertNotEqual(result, 50)
+
+
+Stock matchers
+--------------
+
+testtools comes with many matchers built in. They can all be found in and
+imported from the ``testtools.matchers`` module.
+
+Equals
+~~~~~~
+
+Matches if two items are equal. For example::
+
+ def test_equals_example(self):
+ self.assertThat([42], Equals([42]))
+
+
+Is
+~~~
+
+Matches if two items are identical. For example::
+
+ def test_is_example(self):
+ foo = object()
+ self.assertThat(foo, Is(foo))
+
+
+The raises helper
+~~~~~~~~~~~~~~~~~
+
+Matches if a callable raises a particular type of exception. For example::
+
+ def test_raises_example(self):
+ self.assertThat(lambda: 1/0, raises(ZeroDivisionError))
+
+This is actually a convenience function that combines two other matchers:
+Raises_ and MatchesException_.
+
+
+DocTestMatches
+~~~~~~~~~~~~~~
+
+Matches a string as if it were the output of a doctest_ example. Very useful
+for making assertions about large chunks of text. For example::
+
+ import doctest
+
+ def test_doctest_example(self):
+ output = "Colorless green ideas"
+ self.assertThat(
+ output,
+ DocTestMatches("Colorless ... ideas", doctest.ELLIPSIS))
+
+We highly recommend using the following flags::
+
+ doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF
+
+
+LessThan
+~~~~~~~~
+
+Matches if the given thing is less than the thing in the matcher. For
+example::
+
+ def test_less_than_example(self):
+ self.assertThat(2, LessThan(3))
+
+
+StartsWith, EndsWith
+~~~~~~~~~~~~~~~~~~~~
+
+These matchers check to see if a string starts with or ends with a particular
+substring. For example::
+
+ def test_starts_and_ends_with_example(self):
+ self.assertThat('underground', StartsWith('und'))
+ self.assertThat('underground', EndsWith('und'))
+
+
+MatchesException
+~~~~~~~~~~~~~~~~
+
+Matches an exc_info tuple if the exception is of the correct type. For
+example::
+
+ def test_matches_exception_example(self):
+ try:
+ raise RuntimeError('foo')
+ except RuntimeError:
+ exc_info = sys.exc_info()
+ self.assertThat(exc_info, MatchesException(RuntimeError))
+ self.assertThat(exc_info, MatchesException(RuntimeError('bar'))
+
+Most of the time, you will want to uses `The raises helper`_ instead.
+
+
+NotEquals
+~~~~~~~~~
+
+Matches if something is not equal to something else. Note that this is subtly
+different to ``Not(Equals(x))``. ``NotEquals(x)`` will match if ``y != x``,
+``Not(Equals(x))`` will match if ``not y == x``.
+
+You only need to worry about this distinction if you are testing code that
+relies on badly written overloaded equality operators.
+
+
+Combining matchers
+------------------
+
+One great thing about matchers is that you can readily combine existing
+matchers to get variations on their behaviour or to quickly build more complex
+assertions.
+
+Below are a few of the combining matchers that come with testtools.
+
+
+Not
+~~~
+
+Negates another matcher. For example::
+
+ def test_not_example(self):
+ self.assertThat([42], Not(Equals("potato")))
+ self.assertThat([42], Not(Is([42])))
+
+If you find yourself using ``Not`` frequently, you may wish to create a custom
+matcher for it. For example::
+
+ IsNot = lambda x: Not(Is(x))
+
+ def test_not_example_2(self):
+ self.assertThat([42], IsNot([42]))
+
+
+Annotate
+~~~~~~~~
+
+Used to add custom notes to a matcher. For example::
+
+ def test_annotate_example(self):
+ result = 43
+ self.assertThat(
+ result, Annotate("Not the answer to the Question!", Equals(42))
+
+Since the annotation is only ever displayed when there is a mismatch
+(e.g. when ``result`` does not equal 42), it's a good idea to phrase the note
+negatively, so that it describes what a mismatch actually means.
+
+As with Not_, you may wish to create a custom matcher that describes a
+common operation. For example::
+
+ PoliticallyEquals = lambda x: Annotate("Death to the aristos!", Equals(x))
+
+ def test_annotate_example_2(self):
+ self.assertThat("orange", PoliticallyEquals("yellow"))
+
+
+MatchesAll
+~~~~~~~~~~
+
+Combines many matchers to make a new matcher. The new matcher will only match
+things that match every single one of the component matchers.
+
+It's much easier to understand in Python than in English::
+
+ def test_matches_all_example(self):
+ has_und_at_both_ends = MatchesAll(StartsWith("und"), EndsWith("und"))
+ # This will succeed.
+ self.assertThat("underground", has_und_at_both_ends)
+ # This will fail.
+ self.assertThat("found", has_und_at_both_ends)
+ # So will this.
+ self.assertThat("undead", has_und_at_both_ends)
+
+At this point some people ask themselves, "why bother doing this at all? why
+not just have two separate assertions?". It's a good question.
+
+The first reason is that when a ``MatchesAll`` gets a mismatch, the error will
+include information about all of the bits that mismatched. When you have two
+separate assertions, as below::
+
+ def test_two_separate_assertions(self):
+ self.assertThat("foo", StartsWith("und"))
+ self.assertThat("foo", EndsWith("und"))
+
+Then you get absolutely no information from the second assertion if the first
+assertion fails. Tests are largely there to help you debug code, so having
+more information in error messages is a big help.
+
+The second reason is that it is sometimes useful to give a name to a set of
+matchers. ``has_und_at_both_ends`` is a bit contrived, of course, but it is
+clear.
+
+
+MatchesAny
+~~~~~~~~~~
+
+Like MatchesAll_, ``MatchesAny`` combines many matchers to make a new
+matcher. The difference is that the new matchers will match a thing if it
+matches *any* of the component matchers.
+
+For example::
+
+ def test_matches_any_example(self):
+ self.assertThat(42, MatchesAny(Equals(5), Not(Equals(6))))
+
+
+Raises
+~~~~~~
+
+Takes whatever the callable raises as an exc_info tuple and matches it against
+whatever matcher it was given. For example, if you want to assert that a
+callable raises an exception of a given type::
+
+ def test_raises_example(self):
+ self.assertThat(
+ lambda: 1/0, Raises(MatchesException(ZeroDivisionError)))
+
+Although note that this could also be written as::
+
+ def test_raises_example_convenient(self):
+ self.assertThat(lambda: 1/0, raises(ZeroDivisionError))
+
+See also MatchesException_ and `the raises helper`_
+
+
+Writing your own matchers
+-------------------------
+
+Combining matchers is fun and can get you a very long way indeed, but
+sometimes you will have to write your own. Here's how.
+
+You need to make two closely-linked objects: a ``Matcher`` and a
+``Mismatch``. The ``Matcher`` knows how to actually make the comparison, and
+the ``Mismatch`` knows how describe a failure to match.
+
+Here's an example matcher::
+
+ class IsDivisibleBy(object):
+ """Match if a number is divisible by another number."""
+ def __init__(self, divider):
+ self.divider = divider
+ def __str__(self):
+ return 'IsDivisibleBy(%s)' % (self.divider,)
+ def match(self, actual):
+ remainder = actual % self.divider
+ if remainder != 0:
+ return IsDivisibleByMismatch(actual, self.divider, remainder)
+ else:
+ return None
+
+The matcher has a constructor that takes parameters that describe what you
+actually *expect*, in this case a number that other numbers ought to be
+divisible by. It has a ``__str__`` method, the result of which is displayed
+on failure by ``assertThat`` and a ``match`` method that does the actual
+matching.
+
+``match`` takes something to match against, here ``actual``, and decides
+whether or not it matches. If it does match, then ``match`` must return
+``None``. If it does *not* match, then ``match`` must return a ``Mismatch``
+object. ``assertThat`` will call ``match`` and then fail the test if it
+returns a non-None value. For example::
+
+ def test_is_divisible_by_example(self):
+ # This succeeds, since IsDivisibleBy(5).match(10) returns None.
+ self.assertThat(10, IsDivisbleBy(5))
+ # This fails, since IsDivisibleBy(7).match(10) returns a mismatch.
+ self.assertThat(10, IsDivisbleBy(7))
+
+The mismatch is responsible for what sort of error message the failing test
+generates. Here's an example mismatch::
+
+ class IsDivisibleByMismatch(object):
+ def __init__(self, number, divider, remainder):
+ self.number = number
+ self.divider = divider
+ self.remainder = remainder
+
+ def describe(self):
+ return "%s is not divisible by %s, %s remains" % (
+ self.number, self.divider, self.remainder)
+
+ def get_details(self):
+ return {}
+
+The mismatch takes information about the mismatch, and provides a ``describe``
+method that assembles all of that into a nice error message for end users.
+You can use the ``get_details`` method to provide extra, arbitrary data with
+the mismatch (e.g. the contents of a log file). Most of the time it's fine to
+just return an empty dict. You can read more about Details_ elsewhere in this
+document.
+
+Sometimes you don't need to create a custom mismatch class. In particular, if
+you don't care *when* the description is calculated, then you can just do that
+in the Matcher itself like this::
+
+ def match(self, actual):
+ remainder = actual % self.divider
+ if remainder != 0:
+ return Mismatch(
+ "%s is not divisible by %s, %s remains" % (
+ actual, self.divider, remainder))
+ else:
+ return None
+
+
+Details
+=======
+
+As we may have mentioned once or twice already, one of the great benefits of
+automated tests is that they help find, isolate and debug errors in your
+system.
+
+Frequently however, the information provided by a mere assertion failure is
+not enough. It's often useful to have other information: the contents of log
+files; what queries were run; benchmark timing information; what state certain
+subsystem components are in and so forth.
+
+testtools calls all of these things "details" and provides a single, powerful
+mechanism for including this information in your test run.
+
+Here's an example of how to add them::
+
+ from testtools import TestCase
+ from testtools.content import text_content
+
+ class TestSomething(TestCase):
+
+ def test_thingy(self):
+ self.addDetail('arbitrary-color-name', text_content("blue"))
+ 1 / 0 # Gratuitous error!
+
+A detail an arbitrary piece of content given a name that's unique within the
+test. Here the name is ``arbitrary-color-name`` and the content is
+``text_content("blue")``. The name can be any text string, and the content
+can be any ``testtools.content.Content`` object.
+
+When the test runs, testtools will show you something like this::
+
+ ======================================================================
+ ERROR: exampletest.TestSomething.test_thingy
+ ----------------------------------------------------------------------
+ Text attachment: arbitrary-color-name
+ ------------
+ blue
+ ------------
+ Text attachment: traceback
+ ------------
+ Traceback (most recent call last):
+ ...
+ File "exampletest.py", line 8, in test_thingy
+ 1 / 0 # Gratuitous error!
+ ZeroDivisionError: integer division or modulo by zero
+ ------------
+ Ran 1 test in 0.030s
+
+As you can see, the detail is included as a "Text attachment", here saying
+that our arbitrary-color-name is "blue".
+
+
+Content
+-------
+
+For the actual content of details, testtools uses its own MIME-based Content
+object. This allows you to attach any information that you could possibly
+conceive of to a test, and allows testtools to use or serialize that
+information.
+
+The basic ``testtools.content.Content`` object is constructed from a
+``testtools.content.ContentType`` and a nullary callable that must return an
+iterator of chunks of bytes that the content is made from.
+
+So, to make a Content object that is just a simple string of text, you can
+do::
+
+ from testtools.content import Content
+ from testtools.content_type import ContentType
+
+ text = Content(ContentType('text', 'plain'), lambda: ["some text"])
+
+Because adding small bits of text content is very common, there's also a
+convenience method::
+
+ text = text_content("some text")
+
+To make content out of an image stored on disk, you could do something like::
+
+ image = Content(ContentType('image', 'png'), lambda: open('foo.png').read())
+
+The ``lambda`` helps make sure that the file is opened and the actual bytes
+read only when they are needed â by default, when the test is finished. This
+means that tests can construct and add Content objects freely without worrying
+too much about how they affect run time.
+
+
+A realistic example
+-------------------
+
+A very common use of details is to add a log file to failing tests. Say your
+project has a server represented by a class ``SomeServer`` that you can start
+up and shut down in tests, but runs in another process. You want to test
+interaction with that server, and whenever the interaction fails, you want to
+see the client-side error *and* the logs from the server-side. Here's how you
+might do it::
+
+ from testtools import TestCase
+ from testtools.content import Content
+ from testtools.content_type import UTF8_TEXT
+
+ from myproject import SomeServer
+
+ class SomeTestCase(TestCase):
+
+ def setUp(self):
+ super(SomeTestCase, self).setUp()
+ self.server = SomeServer()
+ self.server.start_up()
+ self.addCleanup(self.server.shut_down)
+ self.addCleanup(self.attach_log_file)
+
+ def attach_log_file(self):
+ self.addDetail(
+ 'log-file',
+ Content(UTF8_TEXT,
+ lambda: open(self.server.logfile, 'r').readlines()))
+
+ def test_a_thing(self):
+ self.assertEqual("cool", self.server.temperature)
+
+This test will attach the log file of ``SomeServer`` to each test that is
+run. testtools will only display the log file for failing tests, so it's not
+such a big deal.
+
+Note the callable passed to ``Content`` reads the log file line-by-line. This
+is something of an optimization. If we naively returned all of the log file
+as one bytestring, ``Content`` would treat that as a list of byte chunks.
+
+If the act of adding at detail is expensive, you might want to use
+addOnException_ so that you only do it when a test actually raises an
+exception.
+
+
+Controlling test execution
+==========================
+
+.. _addCleanup:
+
+addCleanup
+----------
+
+``TestCase.addCleanup`` is a robust way to arrange for a clean up function to
+be called before ``tearDown``. This is a powerful and simple alternative to
+putting clean up logic in a try/finally block or ``tearDown`` method. For
+example::
+
+ def test_foo(self):
+ foo.lock()
+ self.addCleanup(foo.unlock)
+ ...
+
+This is particularly useful if you have some sort of factory in your test::
+
+ def make_locked_foo(self):
+ foo = Foo()
+ foo.lock()
+ self.addCleanup(foo.unlock)
+ return foo
+
+ def test_frotz_a_foo(self):
+ foo = self.make_locked_foo()
+ foo.frotz()
+ self.assertEqual(foo.frotz_count, 1)
+
+Any extra arguments or keyword arguments passed to ``addCleanup`` are passed
+to the callable at cleanup time.
+
+Cleanups can also report multiple errors, if appropriate by wrapping them in
+a ``testtools.MultipleExceptions`` object::
+
+ raise MultipleExceptions(exc_info1, exc_info2)
+
+
+Fixtures
+--------
+
+Tests often depend on a system being set up in a certain way, or having
+certain resources available to them. Perhaps a test needs a connection to the
+database or access to a running external server.
+
+One common way of doing this is to do::
+
+ class SomeTest(TestCase):
+ def setUp(self):
+ super(SomeTest, self).setUp()
+ self.server = Server()
+ self.server.setUp()
+ self.addCleanup(self.server.tearDown)
+
+testtools provides a more convenient, declarative way to do the same thing::
+
+ class SomeTest(TestCase):
+ def setUp(self):
+ super(SomeTest, self).setUp()
+ self.server = self.useFixture(Server())
+
+``useFixture(fixture)`` calls ``setUp`` on the fixture, schedules a clean up
+to clean it up, and schedules a clean up to attach all details_ held by the
+fixture to the test case. The fixture object must meet the
+``fixtures.Fixture`` protocol (version 0.3.4 or newer, see fixtures_).
+
+If you have anything beyond the most simple test set up, we recommend that
+you put this set up into a ``Fixture`` class. Once there, the fixture can be
+easily re-used by other tests and can be combined with other fixtures to make
+more complex resources.
+
+
+Skipping tests
+--------------
+
+Many reasons exist to skip a test: a dependency might be missing; a test might
+be too expensive and thus should not berun while on battery power; or perhaps
+the test is testing an incomplete feature.
+
+``TestCase.skipTest`` is a simple way to have a test stop running and be
+reported as a skipped test, rather than a success, error or failure. For
+example::
+
+ def test_make_symlink(self):
+ symlink = getattr(os, 'symlink', None)
+ if symlink is None:
+ self.skipTest("No symlink support")
+ symlink(whatever, something_else)
+
+Using ``skipTest`` means that you can make decisions about what tests to run
+as late as possible, and close to the actual tests. Without it, you might be
+forced to use convoluted logic during test loading, which is a bit of a mess.
+
+Legacy skip support
+~~~~~~~~~~~~~~~~~~~
+
+If you are using a this feature when running your test suite with a legacy
+``TestResult`` object that is missing the ``addSkip`` method, then the
+``addError`` method will be invoked instead. If you are using a test result
+from testtools, you do not have to worry about this.
+
+In older versions of testtools, ``skipTest`` was known as ``skip``. Since
+Python 2.7 added ``skipTest`` support, the ``skip`` name is now deprecated.
+No warning is emitted yet â some time in the future we may do so.
+
+
+addOnException
+--------------
+
+Sometimes, you might wish to do something only when a test fails. Perhaps you
+need to run expensive diagnostic routines or some such.
+``TestCase.addOnException`` allows you to easily do just this. For example::
+
+ class SomeTest(TestCase):
+ def setUp(self):
+ super(SomeTest, self).setUp()
+ self.server = self.useFixture(SomeServer())
+ self.addOnException(self.attach_server_diagnostics)
+
+ def attach_server_diagnostics(self, exc_info):
+ self.server.prep_for_diagnostics() # Expensive!
+ self.addDetail('server-diagnostics', self.server.get_diagnostics)
+
+ def test_a_thing(self):
+ self.assertEqual('cheese', 'chalk')
+
+In this example, ``attach_server_diagnostics`` will only be called when a test
+fails. It is given the exc_info tuple of the error raised by the test, just
+in case it is needed.
+
+
+Twisted support
+---------------
+
+testtools provides *highly experimental* support for running Twisted tests â
+tests that return a Deferred_ and rely on the Twisted reactor. You should not
+use this feature right now. We reserve the right to change the API and
+behaviour without telling you first.
+
+However, if you are going to, here's how you do it::
+
+ from testtools import TestCase
+ from testtools.deferredruntest import AsynchronousDeferredRunTest
+
+ class MyTwistedTests(TestCase):
+
+ run_tests_with = AsynchronousDeferredRunTest
+
+ def test_foo(self):
+ # ...
+ return d
+
+In particular, note that you do *not* have to use a special base ``TestCase``
+in order to run Twisted tests.
+
+You can also run individual tests within a test case class using the Twisted
+test runner::
+
+ class MyTestsSomeOfWhichAreTwisted(TestCase):
+
+ def test_normal(self):
+ pass
+
+ @run_test_with(AsynchronousDeferredRunTest)
+ def test_twisted(self):
+ # ...
+ return d
+
+Here are some tips for converting your Trial tests into testtools tests.
+
+* Use the ``AsynchronousDeferredRunTest`` runner
+* Make sure to upcall to ``setUp`` and ``tearDown``
+* Don't use ``setUpClass`` or ``tearDownClass``
+* Don't expect setting .todo, .timeout or .skip attributes to do anything
+* ``flushLoggedErrors`` is ``testtools.deferredruntest.flush_logged_errors``
+* ``assertFailure`` is ``testtools.deferredruntest.assert_fails_with``
+* Trial spins the reactor a couple of times before cleaning it up,
+ ``AsynchronousDeferredRunTest`` does not. If you rely on this behavior, use
+ ``AsynchronousDeferredRunTestForBrokenTwisted``.
+
+
+Test helpers
+============
+
+testtools comes with a few little things that make it a little bit easier to
+write tests.
+
+
+TestCase.patch
+--------------
+
+``patch`` is a convenient way to monkey-patch a Python object for the duration
+of your test. It's especially useful for testing legacy code. e.g.::
+
+ def test_foo(self):
+ my_stream = StringIO()
+ self.patch(sys, 'stderr', my_stream)
+ run_some_code_that_prints_to_stderr()
+ self.assertEqual('', my_stream.getvalue())
+
+The call to ``patch`` above masks ``sys.stderr`` with ``my_stream`` so that
+anything printed to stderr will be captured in a StringIO variable that can be
+actually tested. Once the test is done, the real ``sys.stderr`` is restored to
+its rightful place.
+
+
+Creation methods
+----------------
+
+Often when writing unit tests, you want to create an object that is a
+completely normal instance of its type. You don't want there to be anything
+special about its properties, because you are testing generic behaviour rather
+than specific conditions.
+
+A lot of the time, test authors do this by making up silly strings and numbers
+and passing them to constructors (e.g. 42, 'foo', "bar" etc), and that's
+fine. However, sometimes it's useful to be able to create arbitrary objects
+at will, without having to make up silly sample data.
+
+To help with this, ``testtools.TestCase`` implements creation methods called
+``getUniqueString`` and ``getUniqueInteger``. They return strings and
+integers that are unique within the context of the test that can be used to
+assemble more complex objects. Here's a basic example where
+``getUniqueString`` is used instead of saying "foo" or "bar" or whatever::
+
+ class SomeTest(TestCase):
+
+ def test_full_name(self):
+ first_name = self.getUniqueString()
+ last_name = self.getUniqueString()
+ p = Person(first_name, last_name)
+ self.assertEqual(p.full_name, "%s %s" % (first_name, last_name))
+
+
+And here's how it could be used to make a complicated test::
+
+ class TestCoupleLogic(TestCase):
+
+ def make_arbitrary_person(self):
+ return Person(self.getUniqueString(), self.getUniqueString())
+
+ def test_get_invitation(self):
+ a = self.make_arbitrary_person()
+ b = self.make_arbitrary_person()
+ couple = Couple(a, b)
+ event_name = self.getUniqueString()
+ invitation = couple.get_invitation(event_name)
+ self.assertEqual(
+ invitation,
+ "We invite %s and %s to %s" % (
+ a.full_name, b.full_name, event_name))
+
+Essentially, creation methods like these are a way of reducing the number of
+assumptions in your tests and communicating to test readers that the exact
+details of certain variables don't actually matter.
+
+See pages 419-423 of `xUnit Test Patterns`_ by Gerard Meszaros for a detailed
+discussion of creation methods.
+
+
+General helpers
+===============
+
+Conditional imports
+-------------------
+
+Lots of the time we would like to conditionally import modules. testtools
+needs to do this itself, and graciously extends the ability to its users.
+
+Instead of::
+
+ try:
+ from twisted.internet import defer
+ except ImportError:
+ defer = None
+
+You can do::
+
+ defer = try_import('twisted.internet.defer')
+
+
+Instead of::
+
+ try:
+ from StringIO import StringIO
+ except ImportError:
+ from io import StringIO
+
+You can do::
+
+ StringIO = try_imports(['StringIO.StringIO', 'io.StringIO'])
+
+
+.. _testrepository: https://launchpad.net/testrepository
+.. _Trial: http://twistedmatrix.com/documents/current/core/howto/testing.html
+.. _nose: http://somethingaboutorange.com/mrl/projects/nose/
+.. _unittest2: http://pypi.python.org/pypi/unittest2
+.. _zope.testrunner: http://pypi.python.org/pypi/zope.testrunner/
+.. _xUnit test patterns: http://xunitpatterns.com/
+.. _fixtures: http://pypi.python.org/pypi/fixtures
+.. _unittest: http://docs.python.org/library/unittest.html
+.. _doctest: http://docs.python.org/library/doctest.html
+.. _Deferred: http://twistedmatrix.com/documents/current/core/howto/defer.html
+.. _discover: http://pypi.python.org/pypi/discover
+.. _`testtools API docs`: http://mumak.net/testtools/apidocs/
=== renamed file 'HACKING' => 'doc/hacking.rst'
--- HACKING 2010-10-17 12:54:11 +0000
+++ doc/hacking.rst 2010-11-28 15:19:56 +0000
@@ -1,17 +1,15 @@
-===================================
-Notes for contributing to testtools
-===================================
+=========================
+Contributing to testtools
+=========================
Coding style
------------
-In general, follow PEP 8 <http://www.python.org/dev/peps/pep-0008/>.
-
-For consistency with the standard library's ``unittest`` module, method names
-are generally ``camelCase``.
-
-testtools supports Python 2.4 and later, so avoid any 2.5-only features like
-the ``with`` statement.
+In general, follow `PEP 8`_ except where consistency with the standard
+library's unittest_ module would suggest otherwise.
+
+testtools supports Python 2.4 and later, including Python 3, so avoid any
+2.5-only features like the ``with`` statement.
Copyright assignment
@@ -21,10 +19,9 @@
testing code it ships. For that reason we require all contributions (that are
non-trivial) to meet one of the following rules:
- - be inapplicable for inclusion in Python.
- - be able to be included in Python without further contact with the
- contributor.
- - be copyright assigned to Jonathan M. Lange.
+* be inapplicable for inclusion in Python.
+* be able to be included in Python without further contact with the contributor.
+* be copyright assigned to Jonathan M. Lange.
Please pick one of these and specify it when contributing code to testtools.
@@ -33,7 +30,7 @@
---------
All code that is not copyright assigned to Jonathan M. Lange (see Copyright
-Assignment above) needs to be licensed under the MIT license that testtools
+Assignment above) needs to be licensed under the `MIT license`_ that testtools
uses, so that testtools can ship it.
@@ -45,20 +42,31 @@
Take particular care to make sure the *intent* of each test is clear.
-You can run tests with ``make check``, or by running ``./run-tests`` directly.
+You can run tests with ``make check``.
+
+
+Documentation
+-------------
+
+Documents are written using the Sphinx_ variant of reStructuredText_. All
+public methods, functions, classes and modules must have API documentation.
+When changing code, be sure to check the API documentation to see if it could
+be improved. Before submitting changes to trunk, look over them and see if
+the manuals ought to be updated.
Source layout
-------------
The top-level directory contains the ``testtools/`` package directory, and
-miscellaneous files like README and setup.py.
+miscellaneous files like ``README`` and ``setup.py``.
The ``testtools/`` directory is the Python package itself. It is separated
into submodules for internal clarity, but all public APIs should be âpromotedâ
into the top-level package by importing them in ``testtools/__init__.py``.
-Users of testtools should never import a submodule, they are just
-implementation details.
+Users of testtools should never import a submodule in order to use a stable
+API. Unstable APIs like ``testtools.matchers`` and
+``testtools.deferredruntest`` should be exported as submodules.
Tests belong in ``testtools/tests/``.
@@ -84,8 +92,8 @@
branch in launchpad, and submit it for merging to lp:testtools. It will then
be reviewed before it can be merged to trunk. It will be reviewed by someone:
- * not the author
- * a committer (member of the testtools-devs team)
+* not the author
+* a committer (member of the `~testtools-dev`_ team)
As a special exception, while the testtools committers team is small and prone
to blocking, a merge request from a committer that has not been reviewed after
@@ -111,24 +119,32 @@
Release tasks
-------------
- 1. Choose a version number, say X.Y.Z
- 1. Branch from trunk to testtools-X.Y.Z
- 1. In testtools-X.Y.Z, ensure __init__ has version X.Y.Z.
- 1. Replace NEXT in NEWS with the version number X.Y.Z, adjusting the reST.
- 1. Possibly write a blurb into NEWS.
- 1. Replace any additional references to NEXT with the version being
- released. (should be none).
- 1. Commit the changes.
- 1. Tag the release, bzr tag testtools-X.Y.Z
- 1. Create a source distribution and upload to pypi ('make release').
- 1. Make sure all "Fix committed" bugs are in the 'next' milestone on
- Launchpad
- 1. Rename the 'next' milestone on Launchpad to 'X.Y.Z'
- 1. Create a release on the newly-renamed 'X.Y.Z' milestone
- 1. Upload the tarball and asc file to Launchpad
- 1. Merge the release branch testtools-X.Y.Z into trunk. Before the commit,
- add a NEXT heading to the top of NEWS. Push trunk to Launchpad.
- 1. If a new series has been created (e.g. 0.10.0), make the series on Launchpad.
- 1. Make a new milestone for the *next release*.
- 1. During release we rename NEXT to $version.
- 1. We call new milestones NEXT.
+#. Choose a version number, say X.Y.Z
+#. Branch from trunk to testtools-X.Y.Z
+#. In testtools-X.Y.Z, ensure __init__ has version X.Y.Z.
+#. Replace NEXT in NEWS with the version number X.Y.Z, adjusting the reST.
+#. Possibly write a blurb into NEWS.
+#. Replace any additional references to NEXT with the version being released.
+ (should be none).
+#. Commit the changes.
+#. Tag the release, bzr tag testtools-X.Y.Z
+#. Create a source distribution and upload to pypi ('make release').
+#. Make sure all "Fix committed" bugs are in the 'next' milestone on Launchpad
+#. Rename the 'next' milestone on Launchpad to 'X.Y.Z'
+#. Create a release on the newly-renamed 'X.Y.Z' milestone
+#. Upload the tarball and asc file to Launchpad
+#. Merge the release branch testtools-X.Y.Z into trunk. Before the commit,
+ add a NEXT heading to the top of NEWS. Push trunk to Launchpad.
+#. If a new series has been created (e.g. 0.10.0), make the series on Launchpad.
+#. Make a new milestone for the *next release*.
+
+ #. During release we rename NEXT to $version.
+ #. We call new milestones NEXT.
+
+
+.. _PEP 8: http://www.python.org/dev/peps/pep-0008/
+.. _unittest: http://docs.python.org/library/unittest.html
+.. _~testtools-dev: https://launchpad.net/~testtools-dev
+.. _MIT license: http://www.opensource.org/licenses/mit-license.php
+.. _Sphinx: http://sphinx.pocoo.org/
+.. _restructuredtext: http://docutils.sourceforge.net/rst.html
=== added file 'doc/index.rst'
--- doc/index.rst 1970-01-01 00:00:00 +0000
+++ doc/index.rst 2010-11-28 15:19:56 +0000
@@ -0,0 +1,30 @@
+.. testtools documentation master file, created by
+ sphinx-quickstart on Sun Nov 28 13:45:40 2010.
+ You can adapt this file completely to your liking, but it should at least
+ contain the root `toctree` directive.
+
+testtools: tasteful testing for Python
+======================================
+
+testtools is a set of extensions to the Python standard library's unit testing
+framework. These extensions have been derived from many years of experience
+with unit testing in Python and come from many different sources. testtools
+also ports recent unittest changes all the way back to Python 2.4.
+
+
+Contents:
+
+.. toctree::
+ :maxdepth: 2
+
+ for-test-authors
+ for-framework-folk
+ hacking
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+
=== added file 'doc/make.bat'
--- doc/make.bat 1970-01-01 00:00:00 +0000
+++ doc/make.bat 2010-11-28 15:19:56 +0000
@@ -0,0 +1,113 @@
+@ECHO OFF
+
+REM Command file for Sphinx documentation
+
+set SPHINXBUILD=sphinx-build
+set BUILDDIR=_build
+set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .
+if NOT "%PAPER%" == "" (
+ set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
+)
+
+if "%1" == "" goto help
+
+if "%1" == "help" (
+ :help
+ echo.Please use `make ^<target^>` where ^<target^> is one of
+ echo. html to make standalone HTML files
+ echo. dirhtml to make HTML files named index.html in directories
+ echo. pickle to make pickle files
+ echo. json to make JSON files
+ echo. htmlhelp to make HTML files and a HTML help project
+ echo. qthelp to make HTML files and a qthelp project
+ echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter
+ echo. changes to make an overview over all changed/added/deprecated items
+ echo. linkcheck to check all external links for integrity
+ echo. doctest to run all doctests embedded in the documentation if enabled
+ goto end
+)
+
+if "%1" == "clean" (
+ for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
+ del /q /s %BUILDDIR%\*
+ goto end
+)
+
+if "%1" == "html" (
+ %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
+ echo.
+ echo.Build finished. The HTML pages are in %BUILDDIR%/html.
+ goto end
+)
+
+if "%1" == "dirhtml" (
+ %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
+ echo.
+ echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
+ goto end
+)
+
+if "%1" == "pickle" (
+ %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
+ echo.
+ echo.Build finished; now you can process the pickle files.
+ goto end
+)
+
+if "%1" == "json" (
+ %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
+ echo.
+ echo.Build finished; now you can process the JSON files.
+ goto end
+)
+
+if "%1" == "htmlhelp" (
+ %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
+ echo.
+ echo.Build finished; now you can run HTML Help Workshop with the ^
+.hhp project file in %BUILDDIR%/htmlhelp.
+ goto end
+)
+
+if "%1" == "qthelp" (
+ %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
+ echo.
+ echo.Build finished; now you can run "qcollectiongenerator" with the ^
+.qhcp project file in %BUILDDIR%/qthelp, like this:
+ echo.^> qcollectiongenerator %BUILDDIR%\qthelp\testtools.qhcp
+ echo.To view the help file:
+ echo.^> assistant -collectionFile %BUILDDIR%\qthelp\testtools.ghc
+ goto end
+)
+
+if "%1" == "latex" (
+ %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+ echo.
+ echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
+ goto end
+)
+
+if "%1" == "changes" (
+ %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
+ echo.
+ echo.The overview file is in %BUILDDIR%/changes.
+ goto end
+)
+
+if "%1" == "linkcheck" (
+ %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
+ echo.
+ echo.Link check complete; look for any errors in the above output ^
+or in %BUILDDIR%/linkcheck/output.txt.
+ goto end
+)
+
+if "%1" == "doctest" (
+ %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
+ echo.
+ echo.Testing of doctests in the sources finished, look at the ^
+results in %BUILDDIR%/doctest/output.txt.
+ goto end
+)
+
+:end
=== modified file 'setup.py'
--- setup.py 2010-10-18 13:16:02 +0000
+++ setup.py 2010-11-28 15:19:56 +0000
@@ -48,7 +48,8 @@
def get_long_description():
- manual_path = os.path.join(os.path.dirname(__file__), 'MANUAL')
+ manual_path = os.path.join(
+ os.path.dirname(__file__), 'doc/for-test-authors.rst')
return open(manual_path).read()
Follow ups