test_skipping.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530
  1. import unittest
  2. from unittest.test.support import LoggingResult
  3. class Test_TestSkipping(unittest.TestCase):
  4. def test_skipping(self):
  5. class Foo(unittest.TestCase):
  6. def defaultTestResult(self):
  7. return LoggingResult(events)
  8. def test_skip_me(self):
  9. self.skipTest("skip")
  10. events = []
  11. result = LoggingResult(events)
  12. test = Foo("test_skip_me")
  13. self.assertIs(test.run(result), result)
  14. self.assertEqual(events, ['startTest', 'addSkip', 'stopTest'])
  15. self.assertEqual(result.skipped, [(test, "skip")])
  16. events = []
  17. result = test.run()
  18. self.assertEqual(events, ['startTestRun', 'startTest', 'addSkip',
  19. 'stopTest', 'stopTestRun'])
  20. self.assertEqual(result.skipped, [(test, "skip")])
  21. self.assertEqual(result.testsRun, 1)
  22. # Try letting setUp skip the test now.
  23. class Foo(unittest.TestCase):
  24. def defaultTestResult(self):
  25. return LoggingResult(events)
  26. def setUp(self):
  27. self.skipTest("testing")
  28. def test_nothing(self): pass
  29. events = []
  30. result = LoggingResult(events)
  31. test = Foo("test_nothing")
  32. self.assertIs(test.run(result), result)
  33. self.assertEqual(events, ['startTest', 'addSkip', 'stopTest'])
  34. self.assertEqual(result.skipped, [(test, "testing")])
  35. self.assertEqual(result.testsRun, 1)
  36. events = []
  37. result = test.run()
  38. self.assertEqual(events, ['startTestRun', 'startTest', 'addSkip',
  39. 'stopTest', 'stopTestRun'])
  40. self.assertEqual(result.skipped, [(test, "testing")])
  41. self.assertEqual(result.testsRun, 1)
  42. def test_skipping_subtests(self):
  43. class Foo(unittest.TestCase):
  44. def defaultTestResult(self):
  45. return LoggingResult(events)
  46. def test_skip_me(self):
  47. with self.subTest(a=1):
  48. with self.subTest(b=2):
  49. self.skipTest("skip 1")
  50. self.skipTest("skip 2")
  51. self.skipTest("skip 3")
  52. events = []
  53. result = LoggingResult(events)
  54. test = Foo("test_skip_me")
  55. self.assertIs(test.run(result), result)
  56. self.assertEqual(events, ['startTest', 'addSkip', 'addSkip',
  57. 'addSkip', 'stopTest'])
  58. self.assertEqual(len(result.skipped), 3)
  59. subtest, msg = result.skipped[0]
  60. self.assertEqual(msg, "skip 1")
  61. self.assertIsInstance(subtest, unittest.TestCase)
  62. self.assertIsNot(subtest, test)
  63. subtest, msg = result.skipped[1]
  64. self.assertEqual(msg, "skip 2")
  65. self.assertIsInstance(subtest, unittest.TestCase)
  66. self.assertIsNot(subtest, test)
  67. self.assertEqual(result.skipped[2], (test, "skip 3"))
  68. events = []
  69. result = test.run()
  70. self.assertEqual(events,
  71. ['startTestRun', 'startTest', 'addSkip', 'addSkip',
  72. 'addSkip', 'stopTest', 'stopTestRun'])
  73. self.assertEqual([msg for subtest, msg in result.skipped],
  74. ['skip 1', 'skip 2', 'skip 3'])
  75. def test_skipping_decorators(self):
  76. op_table = ((unittest.skipUnless, False, True),
  77. (unittest.skipIf, True, False))
  78. for deco, do_skip, dont_skip in op_table:
  79. class Foo(unittest.TestCase):
  80. def defaultTestResult(self):
  81. return LoggingResult(events)
  82. @deco(do_skip, "testing")
  83. def test_skip(self): pass
  84. @deco(dont_skip, "testing")
  85. def test_dont_skip(self): pass
  86. test_do_skip = Foo("test_skip")
  87. test_dont_skip = Foo("test_dont_skip")
  88. suite = unittest.TestSuite([test_do_skip, test_dont_skip])
  89. events = []
  90. result = LoggingResult(events)
  91. self.assertIs(suite.run(result), result)
  92. self.assertEqual(len(result.skipped), 1)
  93. expected = ['startTest', 'addSkip', 'stopTest',
  94. 'startTest', 'addSuccess', 'stopTest']
  95. self.assertEqual(events, expected)
  96. self.assertEqual(result.testsRun, 2)
  97. self.assertEqual(result.skipped, [(test_do_skip, "testing")])
  98. self.assertTrue(result.wasSuccessful())
  99. events = []
  100. result = test_do_skip.run()
  101. self.assertEqual(events, ['startTestRun', 'startTest', 'addSkip',
  102. 'stopTest', 'stopTestRun'])
  103. self.assertEqual(result.skipped, [(test_do_skip, "testing")])
  104. events = []
  105. result = test_dont_skip.run()
  106. self.assertEqual(events, ['startTestRun', 'startTest', 'addSuccess',
  107. 'stopTest', 'stopTestRun'])
  108. self.assertEqual(result.skipped, [])
  109. def test_skip_class(self):
  110. @unittest.skip("testing")
  111. class Foo(unittest.TestCase):
  112. def defaultTestResult(self):
  113. return LoggingResult(events)
  114. def test_1(self):
  115. record.append(1)
  116. events = []
  117. record = []
  118. result = LoggingResult(events)
  119. test = Foo("test_1")
  120. suite = unittest.TestSuite([test])
  121. self.assertIs(suite.run(result), result)
  122. self.assertEqual(events, ['startTest', 'addSkip', 'stopTest'])
  123. self.assertEqual(result.skipped, [(test, "testing")])
  124. self.assertEqual(record, [])
  125. events = []
  126. result = test.run()
  127. self.assertEqual(events, ['startTestRun', 'startTest', 'addSkip',
  128. 'stopTest', 'stopTestRun'])
  129. self.assertEqual(result.skipped, [(test, "testing")])
  130. self.assertEqual(record, [])
  131. def test_skip_non_unittest_class(self):
  132. @unittest.skip("testing")
  133. class Mixin:
  134. def test_1(self):
  135. record.append(1)
  136. class Foo(Mixin, unittest.TestCase):
  137. pass
  138. record = []
  139. result = unittest.TestResult()
  140. test = Foo("test_1")
  141. suite = unittest.TestSuite([test])
  142. self.assertIs(suite.run(result), result)
  143. self.assertEqual(result.skipped, [(test, "testing")])
  144. self.assertEqual(record, [])
  145. def test_skip_in_setup(self):
  146. class Foo(unittest.TestCase):
  147. def setUp(self):
  148. self.skipTest("skip")
  149. def test_skip_me(self):
  150. self.fail("shouldn't come here")
  151. events = []
  152. result = LoggingResult(events)
  153. test = Foo("test_skip_me")
  154. self.assertIs(test.run(result), result)
  155. self.assertEqual(events, ['startTest', 'addSkip', 'stopTest'])
  156. self.assertEqual(result.skipped, [(test, "skip")])
  157. def test_skip_in_cleanup(self):
  158. class Foo(unittest.TestCase):
  159. def test_skip_me(self):
  160. pass
  161. def tearDown(self):
  162. self.skipTest("skip")
  163. events = []
  164. result = LoggingResult(events)
  165. test = Foo("test_skip_me")
  166. self.assertIs(test.run(result), result)
  167. self.assertEqual(events, ['startTest', 'addSkip', 'stopTest'])
  168. self.assertEqual(result.skipped, [(test, "skip")])
  169. def test_failure_and_skip_in_cleanup(self):
  170. class Foo(unittest.TestCase):
  171. def test_skip_me(self):
  172. self.fail("fail")
  173. def tearDown(self):
  174. self.skipTest("skip")
  175. events = []
  176. result = LoggingResult(events)
  177. test = Foo("test_skip_me")
  178. self.assertIs(test.run(result), result)
  179. self.assertEqual(events, ['startTest', 'addSkip', 'addFailure', 'stopTest'])
  180. self.assertEqual(result.skipped, [(test, "skip")])
  181. def test_skipping_and_fail_in_cleanup(self):
  182. class Foo(unittest.TestCase):
  183. def test_skip_me(self):
  184. self.skipTest("skip")
  185. def tearDown(self):
  186. self.fail("fail")
  187. events = []
  188. result = LoggingResult(events)
  189. test = Foo("test_skip_me")
  190. self.assertIs(test.run(result), result)
  191. self.assertEqual(events, ['startTest', 'addSkip', 'addFailure', 'stopTest'])
  192. self.assertEqual(result.skipped, [(test, "skip")])
  193. def test_expected_failure(self):
  194. class Foo(unittest.TestCase):
  195. @unittest.expectedFailure
  196. def test_die(self):
  197. self.fail("help me!")
  198. events = []
  199. result = LoggingResult(events)
  200. test = Foo("test_die")
  201. self.assertIs(test.run(result), result)
  202. self.assertEqual(events,
  203. ['startTest', 'addExpectedFailure', 'stopTest'])
  204. self.assertFalse(result.failures)
  205. self.assertEqual(result.expectedFailures[0][0], test)
  206. self.assertFalse(result.unexpectedSuccesses)
  207. self.assertTrue(result.wasSuccessful())
  208. def test_expected_failure_with_wrapped_class(self):
  209. @unittest.expectedFailure
  210. class Foo(unittest.TestCase):
  211. def test_1(self):
  212. self.assertTrue(False)
  213. events = []
  214. result = LoggingResult(events)
  215. test = Foo("test_1")
  216. self.assertIs(test.run(result), result)
  217. self.assertEqual(events,
  218. ['startTest', 'addExpectedFailure', 'stopTest'])
  219. self.assertFalse(result.failures)
  220. self.assertEqual(result.expectedFailures[0][0], test)
  221. self.assertFalse(result.unexpectedSuccesses)
  222. self.assertTrue(result.wasSuccessful())
  223. def test_expected_failure_with_wrapped_subclass(self):
  224. class Foo(unittest.TestCase):
  225. def test_1(self):
  226. self.assertTrue(False)
  227. @unittest.expectedFailure
  228. class Bar(Foo):
  229. pass
  230. events = []
  231. result = LoggingResult(events)
  232. test = Bar("test_1")
  233. self.assertIs(test.run(result), result)
  234. self.assertEqual(events,
  235. ['startTest', 'addExpectedFailure', 'stopTest'])
  236. self.assertFalse(result.failures)
  237. self.assertEqual(result.expectedFailures[0][0], test)
  238. self.assertFalse(result.unexpectedSuccesses)
  239. self.assertTrue(result.wasSuccessful())
  240. def test_expected_failure_subtests(self):
  241. # A failure in any subtest counts as the expected failure of the
  242. # whole test.
  243. class Foo(unittest.TestCase):
  244. @unittest.expectedFailure
  245. def test_die(self):
  246. with self.subTest():
  247. # This one succeeds
  248. pass
  249. with self.subTest():
  250. self.fail("help me!")
  251. with self.subTest():
  252. # This one doesn't get executed
  253. self.fail("shouldn't come here")
  254. events = []
  255. result = LoggingResult(events)
  256. test = Foo("test_die")
  257. self.assertIs(test.run(result), result)
  258. self.assertEqual(events,
  259. ['startTest', 'addSubTestSuccess',
  260. 'addExpectedFailure', 'stopTest'])
  261. self.assertFalse(result.failures)
  262. self.assertEqual(len(result.expectedFailures), 1)
  263. self.assertIs(result.expectedFailures[0][0], test)
  264. self.assertFalse(result.unexpectedSuccesses)
  265. self.assertTrue(result.wasSuccessful())
  266. def test_expected_failure_and_fail_in_cleanup(self):
  267. class Foo(unittest.TestCase):
  268. @unittest.expectedFailure
  269. def test_die(self):
  270. self.fail("help me!")
  271. def tearDown(self):
  272. self.fail("bad tearDown")
  273. events = []
  274. result = LoggingResult(events)
  275. test = Foo("test_die")
  276. self.assertIs(test.run(result), result)
  277. self.assertEqual(events,
  278. ['startTest', 'addFailure', 'stopTest'])
  279. self.assertEqual(len(result.failures), 1)
  280. self.assertIn('AssertionError: bad tearDown', result.failures[0][1])
  281. self.assertFalse(result.expectedFailures)
  282. self.assertFalse(result.unexpectedSuccesses)
  283. self.assertFalse(result.wasSuccessful())
  284. def test_expected_failure_and_skip_in_cleanup(self):
  285. class Foo(unittest.TestCase):
  286. @unittest.expectedFailure
  287. def test_die(self):
  288. self.fail("help me!")
  289. def tearDown(self):
  290. self.skipTest("skip")
  291. events = []
  292. result = LoggingResult(events)
  293. test = Foo("test_die")
  294. self.assertIs(test.run(result), result)
  295. self.assertEqual(events,
  296. ['startTest', 'addSkip', 'stopTest'])
  297. self.assertFalse(result.failures)
  298. self.assertFalse(result.expectedFailures)
  299. self.assertFalse(result.unexpectedSuccesses)
  300. self.assertEqual(result.skipped, [(test, "skip")])
  301. self.assertTrue(result.wasSuccessful())
  302. def test_unexpected_success(self):
  303. class Foo(unittest.TestCase):
  304. @unittest.expectedFailure
  305. def test_die(self):
  306. pass
  307. events = []
  308. result = LoggingResult(events)
  309. test = Foo("test_die")
  310. self.assertIs(test.run(result), result)
  311. self.assertEqual(events,
  312. ['startTest', 'addUnexpectedSuccess', 'stopTest'])
  313. self.assertFalse(result.failures)
  314. self.assertFalse(result.expectedFailures)
  315. self.assertEqual(result.unexpectedSuccesses, [test])
  316. self.assertFalse(result.wasSuccessful())
  317. def test_unexpected_success_subtests(self):
  318. # Success in all subtests counts as the unexpected success of
  319. # the whole test.
  320. class Foo(unittest.TestCase):
  321. @unittest.expectedFailure
  322. def test_die(self):
  323. with self.subTest():
  324. # This one succeeds
  325. pass
  326. with self.subTest():
  327. # So does this one
  328. pass
  329. events = []
  330. result = LoggingResult(events)
  331. test = Foo("test_die")
  332. self.assertIs(test.run(result), result)
  333. self.assertEqual(events,
  334. ['startTest',
  335. 'addSubTestSuccess', 'addSubTestSuccess',
  336. 'addUnexpectedSuccess', 'stopTest'])
  337. self.assertFalse(result.failures)
  338. self.assertFalse(result.expectedFailures)
  339. self.assertEqual(result.unexpectedSuccesses, [test])
  340. self.assertFalse(result.wasSuccessful())
  341. def test_unexpected_success_and_fail_in_cleanup(self):
  342. class Foo(unittest.TestCase):
  343. @unittest.expectedFailure
  344. def test_die(self):
  345. pass
  346. def tearDown(self):
  347. self.fail("bad tearDown")
  348. events = []
  349. result = LoggingResult(events)
  350. test = Foo("test_die")
  351. self.assertIs(test.run(result), result)
  352. self.assertEqual(events,
  353. ['startTest', 'addFailure', 'stopTest'])
  354. self.assertEqual(len(result.failures), 1)
  355. self.assertIn('AssertionError: bad tearDown', result.failures[0][1])
  356. self.assertFalse(result.expectedFailures)
  357. self.assertFalse(result.unexpectedSuccesses)
  358. self.assertFalse(result.wasSuccessful())
  359. def test_unexpected_success_and_skip_in_cleanup(self):
  360. class Foo(unittest.TestCase):
  361. @unittest.expectedFailure
  362. def test_die(self):
  363. pass
  364. def tearDown(self):
  365. self.skipTest("skip")
  366. events = []
  367. result = LoggingResult(events)
  368. test = Foo("test_die")
  369. self.assertIs(test.run(result), result)
  370. self.assertEqual(events,
  371. ['startTest', 'addSkip', 'stopTest'])
  372. self.assertFalse(result.failures)
  373. self.assertFalse(result.expectedFailures)
  374. self.assertFalse(result.unexpectedSuccesses)
  375. self.assertEqual(result.skipped, [(test, "skip")])
  376. self.assertTrue(result.wasSuccessful())
  377. def test_skip_doesnt_run_setup(self):
  378. class Foo(unittest.TestCase):
  379. wasSetUp = False
  380. wasTornDown = False
  381. def setUp(self):
  382. Foo.wasSetUp = True
  383. def tornDown(self):
  384. Foo.wasTornDown = True
  385. @unittest.skip('testing')
  386. def test_1(self):
  387. pass
  388. result = unittest.TestResult()
  389. test = Foo("test_1")
  390. suite = unittest.TestSuite([test])
  391. self.assertIs(suite.run(result), result)
  392. self.assertEqual(result.skipped, [(test, "testing")])
  393. self.assertFalse(Foo.wasSetUp)
  394. self.assertFalse(Foo.wasTornDown)
  395. def test_decorated_skip(self):
  396. def decorator(func):
  397. def inner(*a):
  398. return func(*a)
  399. return inner
  400. class Foo(unittest.TestCase):
  401. @decorator
  402. @unittest.skip('testing')
  403. def test_1(self):
  404. pass
  405. result = unittest.TestResult()
  406. test = Foo("test_1")
  407. suite = unittest.TestSuite([test])
  408. self.assertIs(suite.run(result), result)
  409. self.assertEqual(result.skipped, [(test, "testing")])
  410. def test_skip_without_reason(self):
  411. class Foo(unittest.TestCase):
  412. @unittest.skip
  413. def test_1(self):
  414. pass
  415. result = unittest.TestResult()
  416. test = Foo("test_1")
  417. suite = unittest.TestSuite([test])
  418. self.assertIs(suite.run(result), result)
  419. self.assertEqual(result.skipped, [(test, "")])
  420. def test_debug_skipping(self):
  421. class Foo(unittest.TestCase):
  422. def setUp(self):
  423. events.append("setUp")
  424. def tearDown(self):
  425. events.append("tearDown")
  426. def test1(self):
  427. self.skipTest('skipping exception')
  428. events.append("test1")
  429. @unittest.skip("skipping decorator")
  430. def test2(self):
  431. events.append("test2")
  432. events = []
  433. test = Foo("test1")
  434. with self.assertRaises(unittest.SkipTest) as cm:
  435. test.debug()
  436. self.assertIn("skipping exception", str(cm.exception))
  437. self.assertEqual(events, ["setUp"])
  438. events = []
  439. test = Foo("test2")
  440. with self.assertRaises(unittest.SkipTest) as cm:
  441. test.debug()
  442. self.assertIn("skipping decorator", str(cm.exception))
  443. self.assertEqual(events, [])
  444. def test_debug_skipping_class(self):
  445. @unittest.skip("testing")
  446. class Foo(unittest.TestCase):
  447. def setUp(self):
  448. events.append("setUp")
  449. def tearDown(self):
  450. events.append("tearDown")
  451. def test(self):
  452. events.append("test")
  453. events = []
  454. test = Foo("test")
  455. with self.assertRaises(unittest.SkipTest) as cm:
  456. test.debug()
  457. self.assertIn("testing", str(cm.exception))
  458. self.assertEqual(events, [])
  459. def test_debug_skipping_subtests(self):
  460. class Foo(unittest.TestCase):
  461. def setUp(self):
  462. events.append("setUp")
  463. def tearDown(self):
  464. events.append("tearDown")
  465. def test(self):
  466. with self.subTest(a=1):
  467. events.append('subtest')
  468. self.skipTest("skip subtest")
  469. events.append('end subtest')
  470. events.append('end test')
  471. events = []
  472. result = LoggingResult(events)
  473. test = Foo("test")
  474. with self.assertRaises(unittest.SkipTest) as cm:
  475. test.debug()
  476. self.assertIn("skip subtest", str(cm.exception))
  477. self.assertEqual(events, ['setUp', 'subtest'])
  478. if __name__ == "__main__":
  479. unittest.main()