result.py 8.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256
  1. """Test result object"""
  2. import io
  3. import sys
  4. import traceback
  5. from . import util
  6. from functools import wraps
  7. __unittest = True
  8. def failfast(method):
  9. @wraps(method)
  10. def inner(self, *args, **kw):
  11. if getattr(self, 'failfast', False):
  12. self.stop()
  13. return method(self, *args, **kw)
  14. return inner
  15. STDOUT_LINE = '\nStdout:\n%s'
  16. STDERR_LINE = '\nStderr:\n%s'
  17. class TestResult(object):
  18. """Holder for test result information.
  19. Test results are automatically managed by the TestCase and TestSuite
  20. classes, and do not need to be explicitly manipulated by writers of tests.
  21. Each instance holds the total number of tests run, and collections of
  22. failures and errors that occurred among those test runs. The collections
  23. contain tuples of (testcase, exceptioninfo), where exceptioninfo is the
  24. formatted traceback of the error that occurred.
  25. """
  26. _previousTestClass = None
  27. _testRunEntered = False
  28. _moduleSetUpFailed = False
  29. def __init__(self, stream=None, descriptions=None, verbosity=None):
  30. self.failfast = False
  31. self.failures = []
  32. self.errors = []
  33. self.testsRun = 0
  34. self.skipped = []
  35. self.expectedFailures = []
  36. self.unexpectedSuccesses = []
  37. self.collectedDurations = []
  38. self.shouldStop = False
  39. self.buffer = False
  40. self.tb_locals = False
  41. self._stdout_buffer = None
  42. self._stderr_buffer = None
  43. self._original_stdout = sys.stdout
  44. self._original_stderr = sys.stderr
  45. self._mirrorOutput = False
  46. def printErrors(self):
  47. "Called by TestRunner after test run"
  48. def startTest(self, test):
  49. "Called when the given test is about to be run"
  50. self.testsRun += 1
  51. self._mirrorOutput = False
  52. self._setupStdout()
  53. def _setupStdout(self):
  54. if self.buffer:
  55. if self._stderr_buffer is None:
  56. self._stderr_buffer = io.StringIO()
  57. self._stdout_buffer = io.StringIO()
  58. sys.stdout = self._stdout_buffer
  59. sys.stderr = self._stderr_buffer
  60. def startTestRun(self):
  61. """Called once before any tests are executed.
  62. See startTest for a method called before each test.
  63. """
  64. def stopTest(self, test):
  65. """Called when the given test has been run"""
  66. self._restoreStdout()
  67. self._mirrorOutput = False
  68. def _restoreStdout(self):
  69. if self.buffer:
  70. if self._mirrorOutput:
  71. output = sys.stdout.getvalue()
  72. error = sys.stderr.getvalue()
  73. if output:
  74. if not output.endswith('\n'):
  75. output += '\n'
  76. self._original_stdout.write(STDOUT_LINE % output)
  77. if error:
  78. if not error.endswith('\n'):
  79. error += '\n'
  80. self._original_stderr.write(STDERR_LINE % error)
  81. sys.stdout = self._original_stdout
  82. sys.stderr = self._original_stderr
  83. self._stdout_buffer.seek(0)
  84. self._stdout_buffer.truncate()
  85. self._stderr_buffer.seek(0)
  86. self._stderr_buffer.truncate()
  87. def stopTestRun(self):
  88. """Called once after all tests are executed.
  89. See stopTest for a method called after each test.
  90. """
  91. @failfast
  92. def addError(self, test, err):
  93. """Called when an error has occurred. 'err' is a tuple of values as
  94. returned by sys.exc_info().
  95. """
  96. self.errors.append((test, self._exc_info_to_string(err, test)))
  97. self._mirrorOutput = True
  98. @failfast
  99. def addFailure(self, test, err):
  100. """Called when an error has occurred. 'err' is a tuple of values as
  101. returned by sys.exc_info()."""
  102. self.failures.append((test, self._exc_info_to_string(err, test)))
  103. self._mirrorOutput = True
  104. def addSubTest(self, test, subtest, err):
  105. """Called at the end of a subtest.
  106. 'err' is None if the subtest ended successfully, otherwise it's a
  107. tuple of values as returned by sys.exc_info().
  108. """
  109. # By default, we don't do anything with successful subtests, but
  110. # more sophisticated test results might want to record them.
  111. if err is not None:
  112. if getattr(self, 'failfast', False):
  113. self.stop()
  114. if issubclass(err[0], test.failureException):
  115. errors = self.failures
  116. else:
  117. errors = self.errors
  118. errors.append((subtest, self._exc_info_to_string(err, test)))
  119. self._mirrorOutput = True
  120. def addSuccess(self, test):
  121. "Called when a test has completed successfully"
  122. pass
  123. def addSkip(self, test, reason):
  124. """Called when a test is skipped."""
  125. self.skipped.append((test, reason))
  126. def addExpectedFailure(self, test, err):
  127. """Called when an expected failure/error occurred."""
  128. self.expectedFailures.append(
  129. (test, self._exc_info_to_string(err, test)))
  130. @failfast
  131. def addUnexpectedSuccess(self, test):
  132. """Called when a test was expected to fail, but succeed."""
  133. self.unexpectedSuccesses.append(test)
  134. def addDuration(self, test, elapsed):
  135. """Called when a test finished to run, regardless of its outcome.
  136. *test* is the test case corresponding to the test method.
  137. *elapsed* is the time represented in seconds, and it includes the
  138. execution of cleanup functions.
  139. """
  140. # support for a TextTestRunner using an old TestResult class
  141. if hasattr(self, "collectedDurations"):
  142. # Pass test repr and not the test object itself to avoid resources leak
  143. self.collectedDurations.append((str(test), elapsed))
  144. def wasSuccessful(self):
  145. """Tells whether or not this result was a success."""
  146. # The hasattr check is for test_result's OldResult test. That
  147. # way this method works on objects that lack the attribute.
  148. # (where would such result instances come from? old stored pickles?)
  149. return ((len(self.failures) == len(self.errors) == 0) and
  150. (not hasattr(self, 'unexpectedSuccesses') or
  151. len(self.unexpectedSuccesses) == 0))
  152. def stop(self):
  153. """Indicates that the tests should be aborted."""
  154. self.shouldStop = True
  155. def _exc_info_to_string(self, err, test):
  156. """Converts a sys.exc_info()-style tuple of values into a string."""
  157. exctype, value, tb = err
  158. tb = self._clean_tracebacks(exctype, value, tb, test)
  159. tb_e = traceback.TracebackException(
  160. exctype, value, tb,
  161. capture_locals=self.tb_locals, compact=True)
  162. msgLines = list(tb_e.format())
  163. if self.buffer:
  164. output = sys.stdout.getvalue()
  165. error = sys.stderr.getvalue()
  166. if output:
  167. if not output.endswith('\n'):
  168. output += '\n'
  169. msgLines.append(STDOUT_LINE % output)
  170. if error:
  171. if not error.endswith('\n'):
  172. error += '\n'
  173. msgLines.append(STDERR_LINE % error)
  174. return ''.join(msgLines)
  175. def _clean_tracebacks(self, exctype, value, tb, test):
  176. ret = None
  177. first = True
  178. excs = [(exctype, value, tb)]
  179. seen = {id(value)} # Detect loops in chained exceptions.
  180. while excs:
  181. (exctype, value, tb) = excs.pop()
  182. # Skip test runner traceback levels
  183. while tb and self._is_relevant_tb_level(tb):
  184. tb = tb.tb_next
  185. # Skip assert*() traceback levels
  186. if exctype is test.failureException:
  187. self._remove_unittest_tb_frames(tb)
  188. if first:
  189. ret = tb
  190. first = False
  191. else:
  192. value.__traceback__ = tb
  193. if value is not None:
  194. for c in (value.__cause__, value.__context__):
  195. if c is not None and id(c) not in seen:
  196. excs.append((type(c), c, c.__traceback__))
  197. seen.add(id(c))
  198. return ret
  199. def _is_relevant_tb_level(self, tb):
  200. return '__unittest' in tb.tb_frame.f_globals
  201. def _remove_unittest_tb_frames(self, tb):
  202. '''Truncates usercode tb at the first unittest frame.
  203. If the first frame of the traceback is in user code,
  204. the prefix up to the first unittest frame is returned.
  205. If the first frame is already in the unittest module,
  206. the traceback is not modified.
  207. '''
  208. prev = None
  209. while tb and not self._is_relevant_tb_level(tb):
  210. prev = tb
  211. tb = tb.tb_next
  212. if prev is not None:
  213. prev.tb_next = None
  214. def __repr__(self):
  215. return ("<%s run=%i errors=%i failures=%i>" %
  216. (util.strclass(self.__class__), self.testsRun, len(self.errors),
  217. len(self.failures)))