Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Appearance settings

Commit ecb09a8

Browse filesBrowse files
ambvsobolevn
andauthored
[3.11] bpo-46523: fix tests rerun when setUp[Class|Module] fails (GH-30895) (GH-103342)
(cherry picked from commit 9953860) Co-authored-by: Nikita Sobolev <mail@sobolevn.me>
1 parent 8740fd8 commit ecb09a8
Copy full SHA for ecb09a8

File tree

Expand file treeCollapse file tree

3 files changed

+194
-3
lines changed
Filter options
Expand file treeCollapse file tree

3 files changed

+194
-3
lines changed

‎Lib/test/libregrtest/main.py

Copy file name to clipboardExpand all lines: Lib/test/libregrtest/main.py
+34-2Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,19 @@
2828
# Must be smaller than buildbot "1200 seconds without output" limit.
2929
EXIT_TIMEOUT = 120.0
3030

31+
# gh-90681: When rerunning tests, we might need to rerun the whole
32+
# class or module suite if some its life-cycle hooks fail.
33+
# Test level hooks are not affected.
34+
_TEST_LIFECYCLE_HOOKS = frozenset((
35+
'setUpClass', 'tearDownClass',
36+
'setUpModule', 'tearDownModule',
37+
))
38+
39+
EXITCODE_BAD_TEST = 2
40+
EXITCODE_INTERRUPTED = 130
41+
EXITCODE_ENV_CHANGED = 3
42+
EXITCODE_NO_TESTS_RAN = 4
43+
3144

3245
class Regrtest:
3346
"""Execute a test suite.
@@ -331,8 +344,12 @@ def rerun_failed_tests(self):
331344

332345
errors = result.errors or []
333346
failures = result.failures or []
334-
error_names = [test_full_name.split(" ")[0] for (test_full_name, *_) in errors]
335-
failure_names = [test_full_name.split(" ")[0] for (test_full_name, *_) in failures]
347+
error_names = [
348+
self.normalize_test_name(test_full_name, is_error=True)
349+
for (test_full_name, *_) in errors]
350+
failure_names = [
351+
self.normalize_test_name(test_full_name)
352+
for (test_full_name, *_) in failures]
336353
self.ns.verbose = True
337354
orig_match_tests = self.ns.match_tests
338355
if errors or failures:
@@ -358,6 +375,21 @@ def rerun_failed_tests(self):
358375

359376
self.display_result()
360377

378+
def normalize_test_name(self, test_full_name, *, is_error=False):
379+
short_name = test_full_name.split(" ")[0]
380+
if is_error and short_name in _TEST_LIFECYCLE_HOOKS:
381+
# This means that we have a failure in a life-cycle hook,
382+
# we need to rerun the whole module or class suite.
383+
# Basically the error looks like this:
384+
# ERROR: setUpClass (test.test_reg_ex.RegTest)
385+
# or
386+
# ERROR: setUpModule (test.test_reg_ex)
387+
# So, we need to parse the class / module name.
388+
lpar = test_full_name.index('(')
389+
rpar = test_full_name.index(')')
390+
return test_full_name[lpar + 1: rpar].split('.')[-1]
391+
return short_name
392+
361393
def display_result(self):
362394
# If running the test suite for PGO then no one cares about results.
363395
if self.ns.pgo:

‎Lib/test/support/__init__.py

Copy file name to clipboardExpand all lines: Lib/test/support/__init__.py
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1097,7 +1097,7 @@ def _run_suite(suite):
10971097
if junit_xml_list is not None:
10981098
junit_xml_list.append(result.get_xml_element())
10991099

1100-
if not result.testsRun and not result.skipped:
1100+
if not result.testsRun and not result.skipped and not result.errors:
11011101
raise TestDidNotRun
11021102
if not result.wasSuccessful():
11031103
if len(result.errors) == 1 and not result.failures:

‎Lib/test/test_regrtest.py

Copy file name to clipboardExpand all lines: Lib/test/test_regrtest.py
+159Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,11 @@
3030
ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR))
3131
LOG_PREFIX = r'[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?'
3232

33+
EXITCODE_BAD_TEST = 2
34+
EXITCODE_ENV_CHANGED = 3
35+
EXITCODE_NO_TESTS_RAN = 4
36+
EXITCODE_INTERRUPTED = 130
37+
3338
TEST_INTERRUPTED = textwrap.dedent("""
3439
from signal import SIGINT, raise_signal
3540
try:
@@ -1115,6 +1120,160 @@ def test_fail_once(self):
11151120
self.check_executed_tests(output, [testname],
11161121
rerun={testname: "test_fail_once"})
11171122

1123+
def test_rerun_setup_class_hook_failure(self):
1124+
# FAILURE then FAILURE
1125+
code = textwrap.dedent("""
1126+
import unittest
1127+
1128+
class ExampleTests(unittest.TestCase):
1129+
@classmethod
1130+
def setUpClass(self):
1131+
raise RuntimeError('Fail')
1132+
1133+
def test_success(self):
1134+
return
1135+
""")
1136+
testname = self.create_test(code=code)
1137+
1138+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1139+
self.check_executed_tests(output, testname,
1140+
failed=[testname],
1141+
rerun={testname: "ExampleTests"})
1142+
1143+
def test_rerun_teardown_class_hook_failure(self):
1144+
# FAILURE then FAILURE
1145+
code = textwrap.dedent("""
1146+
import unittest
1147+
1148+
class ExampleTests(unittest.TestCase):
1149+
@classmethod
1150+
def tearDownClass(self):
1151+
raise RuntimeError('Fail')
1152+
1153+
def test_success(self):
1154+
return
1155+
""")
1156+
testname = self.create_test(code=code)
1157+
1158+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1159+
self.check_executed_tests(output, testname,
1160+
failed=[testname],
1161+
rerun={testname: "ExampleTests"})
1162+
1163+
def test_rerun_setup_module_hook_failure(self):
1164+
# FAILURE then FAILURE
1165+
code = textwrap.dedent("""
1166+
import unittest
1167+
1168+
def setUpModule():
1169+
raise RuntimeError('Fail')
1170+
1171+
class ExampleTests(unittest.TestCase):
1172+
def test_success(self):
1173+
return
1174+
""")
1175+
testname = self.create_test(code=code)
1176+
1177+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1178+
self.check_executed_tests(output, testname,
1179+
failed=[testname],
1180+
rerun={testname: testname})
1181+
1182+
def test_rerun_teardown_module_hook_failure(self):
1183+
# FAILURE then FAILURE
1184+
code = textwrap.dedent("""
1185+
import unittest
1186+
1187+
def tearDownModule():
1188+
raise RuntimeError('Fail')
1189+
1190+
class ExampleTests(unittest.TestCase):
1191+
def test_success(self):
1192+
return
1193+
""")
1194+
testname = self.create_test(code=code)
1195+
1196+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1197+
self.check_executed_tests(output, testname,
1198+
failed=[testname],
1199+
rerun={testname: testname})
1200+
1201+
def test_rerun_setup_hook_failure(self):
1202+
# FAILURE then FAILURE
1203+
code = textwrap.dedent("""
1204+
import unittest
1205+
1206+
class ExampleTests(unittest.TestCase):
1207+
def setUp(self):
1208+
raise RuntimeError('Fail')
1209+
1210+
def test_success(self):
1211+
return
1212+
""")
1213+
testname = self.create_test(code=code)
1214+
1215+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1216+
self.check_executed_tests(output, testname,
1217+
failed=[testname],
1218+
rerun={testname: "test_success"})
1219+
1220+
def test_rerun_teardown_hook_failure(self):
1221+
# FAILURE then FAILURE
1222+
code = textwrap.dedent("""
1223+
import unittest
1224+
1225+
class ExampleTests(unittest.TestCase):
1226+
def tearDown(self):
1227+
raise RuntimeError('Fail')
1228+
1229+
def test_success(self):
1230+
return
1231+
""")
1232+
testname = self.create_test(code=code)
1233+
1234+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1235+
self.check_executed_tests(output, testname,
1236+
failed=[testname],
1237+
rerun={testname: "test_success"})
1238+
1239+
def test_rerun_async_setup_hook_failure(self):
1240+
# FAILURE then FAILURE
1241+
code = textwrap.dedent("""
1242+
import unittest
1243+
1244+
class ExampleTests(unittest.IsolatedAsyncioTestCase):
1245+
async def asyncSetUp(self):
1246+
raise RuntimeError('Fail')
1247+
1248+
async def test_success(self):
1249+
return
1250+
""")
1251+
testname = self.create_test(code=code)
1252+
1253+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1254+
self.check_executed_tests(output, testname,
1255+
failed=[testname],
1256+
rerun={testname: "test_success"})
1257+
1258+
def test_rerun_async_teardown_hook_failure(self):
1259+
# FAILURE then FAILURE
1260+
code = textwrap.dedent("""
1261+
import unittest
1262+
1263+
class ExampleTests(unittest.IsolatedAsyncioTestCase):
1264+
async def asyncTearDown(self):
1265+
raise RuntimeError('Fail')
1266+
1267+
async def test_success(self):
1268+
return
1269+
""")
1270+
testname = self.create_test(code=code)
1271+
1272+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1273+
self.check_executed_tests(output, testname,
1274+
failed=[testname],
1275+
rerun={testname: "test_success"})
1276+
11181277
def test_no_tests_ran(self):
11191278
code = textwrap.dedent("""
11201279
import unittest

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.