Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit fea98bf

Browse filesBrowse files
vstinnervstinner
authored
[2.7] bpo-30523, bpo-30764, bpo-30776: Sync regrtest from master (#2444)
* bpo-30523: regrtest --list-cases --match (#2401) * regrtest --list-cases now supports --match and --match-file options. Example: ./python -m test --list-cases -m FileTests test_os * --list-cases now also sets support.verbose to False to prevent messages to stdout when loading test modules. * Add support._match_test() private function. (cherry picked from commit ace56d5) (cherry picked from commit 36946c0) * bpo-30764: regrtest: add --fail-env-changed option (#2402) * bpo-30764: regrtest: change exit code on failure * Exit code 2 if failed tests ("bad") * Exit code 3 if interrupted * bpo-30764: regrtest: add --fail-env-changed option If the option is set, mark a test as failed if it alters the environment, for example if it creates a file without removing it. (cherry picked from commit 63f54c6) (cherry picked from commit 1f33857) * bpo-30776: reduce regrtest -R false positives (#2422) * Change the regrtest --huntrleaks checker to decide if a test file leaks or not. Require that each run leaks at least 1 reference. * Warmup runs are now completely ignored: ignored in the checker test and not used anymore to compute the sum. * Add an unit test for a reference leak. Example of reference differences previously considered a failure (leak) and now considered as success (success, no leak): [3, 0, 0] [0, 1, 0] [8, -8, 1] (cherry picked from commit 48b5c42) (cherry picked from commit e0f8b43)
1 parent 2d77558 commit fea98bf
Copy full SHA for fea98bf

3 files changed

+148-42Lines changed: 148 additions & 42 deletions

File tree

Expand file treeCollapse file tree
Open diff view settings
Filter options
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎Lib/test/regrtest.py‎

Copy file name to clipboardExpand all lines: Lib/test/regrtest.py
+34-6Lines changed: 34 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@
6767
don't execute them
6868
--list-cases -- only write the name of test cases that will be run,
6969
don't execute them
70+
--fail-env-changed -- if a test file alters the environment, mark the test
71+
as failed
7072
7173
7274
Additional Option Details:
@@ -327,7 +329,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
327329
'runleaks', 'huntrleaks=', 'memlimit=', 'randseed=',
328330
'multiprocess=', 'slaveargs=', 'forever', 'header', 'pgo',
329331
'failfast', 'match=', 'testdir=', 'list-tests', 'list-cases',
330-
'coverage', 'matchfile='])
332+
'coverage', 'matchfile=', 'fail-env-changed'])
331333
except getopt.error, msg:
332334
usage(2, msg)
333335

@@ -339,6 +341,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
339341
slaveargs = None
340342
list_tests = False
341343
list_cases_opt = False
344+
fail_env_changed = False
342345
for o, a in opts:
343346
if o in ('-h', '--help'):
344347
usage(0)
@@ -439,6 +442,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
439442
list_tests = True
440443
elif o == '--list-cases':
441444
list_cases_opt = True
445+
elif o == '--fail-env-changed':
446+
fail_env_changed = True
442447
else:
443448
print >>sys.stderr, ("No handler for option {}. Please "
444449
"report this as a bug at http://bugs.python.org.").format(o)
@@ -558,7 +563,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
558563
sys.exit(0)
559564

560565
if list_cases_opt:
561-
list_cases(testdir, selected)
566+
list_cases(testdir, selected, match_tests)
562567
sys.exit(0)
563568

564569
if trace:
@@ -908,11 +913,19 @@ def local_runtest():
908913
result = "FAILURE"
909914
elif interrupted:
910915
result = "INTERRUPTED"
916+
elif environment_changed and fail_env_changed:
917+
result = "ENV CHANGED"
911918
else:
912919
result = "SUCCESS"
913920
print("Tests result: %s" % result)
914921

915-
sys.exit(len(bad) > 0 or interrupted)
922+
if bad:
923+
sys.exit(2)
924+
if interrupted:
925+
sys.exit(130)
926+
if fail_env_changed and environment_changed:
927+
sys.exit(3)
928+
sys.exit(0)
916929

917930

918931
STDTESTS = [
@@ -1310,7 +1323,18 @@ def run_the_test():
13101323
if i >= nwarmup:
13111324
deltas.append(rc_after - rc_before)
13121325
print >> sys.stderr
1313-
if any(deltas):
1326+
1327+
# bpo-30776: Try to ignore false positives:
1328+
#
1329+
# [3, 0, 0]
1330+
# [0, 1, 0]
1331+
# [8, -8, 1]
1332+
#
1333+
# Expected leaks:
1334+
#
1335+
# [5, 5, 6]
1336+
# [10, 1, 1]
1337+
if all(delta >= 1 for delta in deltas):
13141338
msg = '%s leaked %s references, sum=%s' % (test, deltas, sum(deltas))
13151339
print >> sys.stderr, msg
13161340
with open(fname, "a") as refrep:
@@ -1501,9 +1525,13 @@ def _list_cases(suite):
15011525
if isinstance(test, unittest.TestSuite):
15021526
_list_cases(test)
15031527
elif isinstance(test, unittest.TestCase):
1504-
print(test.id())
1528+
if test_support._match_test(test):
1529+
print(test.id())
1530+
1531+
def list_cases(testdir, selected, match_tests):
1532+
test_support.verbose = False
1533+
test_support.match_tests = match_tests
15051534

1506-
def list_cases(testdir, selected):
15071535
skipped = []
15081536
for test in selected:
15091537
abstest = get_abs_module(testdir, test)
Collapse file

‎Lib/test/support/__init__.py‎

Copy file name to clipboardExpand all lines: Lib/test/support/__init__.py
+18-14Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1542,6 +1542,23 @@ def _run_suite(suite):
15421542
raise TestFailed(err)
15431543

15441544

1545+
def _match_test(test):
1546+
global match_tests
1547+
1548+
if match_tests is None:
1549+
return True
1550+
test_id = test.id()
1551+
1552+
for match_test in match_tests:
1553+
if fnmatch.fnmatchcase(test_id, match_test):
1554+
return True
1555+
1556+
for name in test_id.split("."):
1557+
if fnmatch.fnmatchcase(name, match_test):
1558+
return True
1559+
return False
1560+
1561+
15451562
def run_unittest(*classes):
15461563
"""Run tests from unittest.TestCase-derived classes."""
15471564
valid_types = (unittest.TestSuite, unittest.TestCase)
@@ -1556,20 +1573,7 @@ def run_unittest(*classes):
15561573
suite.addTest(cls)
15571574
else:
15581575
suite.addTest(unittest.makeSuite(cls))
1559-
def case_pred(test):
1560-
if match_tests is None:
1561-
return True
1562-
test_id = test.id()
1563-
1564-
for match_test in match_tests:
1565-
if fnmatch.fnmatchcase(test_id, match_test):
1566-
return True
1567-
1568-
for name in test_id.split("."):
1569-
if fnmatch.fnmatchcase(name, match_test):
1570-
return True
1571-
return False
1572-
_filter_suite(suite, case_pred)
1576+
_filter_suite(suite, _match_test)
15731577
_run_suite(suite)
15741578

15751579
#=======================================================================
Collapse file

‎Lib/test/test_regrtest.py‎

Copy file name to clipboardExpand all lines: Lib/test/test_regrtest.py
+96-22Lines changed: 96 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -91,19 +91,19 @@ def parse_executed_tests(self, output):
9191
return list(match.group(1) for match in parser)
9292

9393
def check_executed_tests(self, output, tests, skipped=(), failed=(),
94-
omitted=(), randomize=False, interrupted=False):
94+
env_changed=(), omitted=(),
95+
randomize=False, interrupted=False,
96+
fail_env_changed=False):
9597
if isinstance(tests, str):
9698
tests = [tests]
9799
if isinstance(skipped, str):
98100
skipped = [skipped]
99101
if isinstance(failed, str):
100102
failed = [failed]
103+
if isinstance(env_changed, str):
104+
env_changed = [env_changed]
101105
if isinstance(omitted, str):
102106
omitted = [omitted]
103-
ntest = len(tests)
104-
nskipped = len(skipped)
105-
nfailed = len(failed)
106-
nomitted = len(omitted)
107107

108108
executed = self.parse_executed_tests(output)
109109
if randomize:
@@ -129,11 +129,17 @@ def list_regex(line_format, tests):
129129
regex = list_regex('%s test%s failed', failed)
130130
self.check_line(output, regex)
131131

132+
if env_changed:
133+
regex = list_regex('%s test%s altered the execution environment',
134+
env_changed)
135+
self.check_line(output, regex)
136+
132137
if omitted:
133138
regex = list_regex('%s test%s omitted', omitted)
134139
self.check_line(output, regex)
135140

136-
good = ntest - nskipped - nfailed - nomitted
141+
good = (len(tests) - len(skipped) - len(failed)
142+
- len(omitted) - len(env_changed))
137143
if good:
138144
regex = r'%s test%s OK\.$' % (good, plural(good))
139145
if not skipped and not failed and good > 1:
@@ -143,10 +149,12 @@ def list_regex(line_format, tests):
143149
if interrupted:
144150
self.check_line(output, 'Test suite interrupted by signal SIGINT.')
145151

146-
if nfailed:
152+
if failed:
147153
result = 'FAILURE'
148154
elif interrupted:
149155
result = 'INTERRUPTED'
156+
elif fail_env_changed and env_changed:
157+
result = 'ENV CHANGED'
150158
else:
151159
result = 'SUCCESS'
152160
self.check_line(output, 'Tests result: %s' % result)
@@ -325,7 +333,7 @@ def test_main():
325333
test_failing = self.create_test('failing', code=code)
326334
tests = [test_ok, test_failing]
327335

328-
output = self.run_tests(*tests, exitcode=1)
336+
output = self.run_tests(*tests, exitcode=2)
329337
self.check_executed_tests(output, tests, failed=test_failing)
330338

331339
def test_resources(self):
@@ -394,7 +402,7 @@ def test_fromfile(self):
394402
def test_interrupted(self):
395403
code = TEST_INTERRUPTED
396404
test = self.create_test('sigint', code=code)
397-
output = self.run_tests(test, exitcode=1)
405+
output = self.run_tests(test, exitcode=130)
398406
self.check_executed_tests(output, test, omitted=test,
399407
interrupted=True)
400408

@@ -423,7 +431,7 @@ def test_slow_interrupted(self):
423431
args = ("--slowest", "-j2", test)
424432
else:
425433
args = ("--slowest", test)
426-
output = self.run_tests(*args, exitcode=1)
434+
output = self.run_tests(*args, exitcode=130)
427435
self.check_executed_tests(output, test,
428436
omitted=test, interrupted=True)
429437

@@ -461,24 +469,88 @@ def test_main():
461469
support.run_unittest(ForeverTester)
462470
""")
463471
test = self.create_test('forever', code=code)
464-
output = self.run_tests('--forever', test, exitcode=1)
472+
output = self.run_tests('--forever', test, exitcode=2)
465473
self.check_executed_tests(output, [test]*3, failed=test)
466474

475+
def check_leak(self, code, what):
476+
test = self.create_test('huntrleaks', code=code)
477+
478+
filename = 'reflog.txt'
479+
self.addCleanup(support.unlink, filename)
480+
output = self.run_tests('--huntrleaks', '3:3:', test,
481+
exitcode=2,
482+
stderr=subprocess.STDOUT)
483+
self.check_executed_tests(output, [test], failed=test)
484+
485+
line = 'beginning 6 repetitions\n123456\n......\n'
486+
self.check_line(output, re.escape(line))
487+
488+
line2 = '%s leaked [1, 1, 1] %s, sum=3\n' % (test, what)
489+
self.assertIn(line2, output)
490+
491+
with open(filename) as fp:
492+
reflog = fp.read()
493+
self.assertIn(line2, reflog)
494+
495+
@unittest.skipUnless(Py_DEBUG, 'need a debug build')
496+
def test_huntrleaks(self):
497+
# test --huntrleaks
498+
code = textwrap.dedent("""
499+
import unittest
500+
from test import support
501+
502+
GLOBAL_LIST = []
503+
504+
class RefLeakTest(unittest.TestCase):
505+
def test_leak(self):
506+
GLOBAL_LIST.append(object())
507+
508+
def test_main():
509+
support.run_unittest(RefLeakTest)
510+
""")
511+
self.check_leak(code, 'references')
512+
467513
def test_list_tests(self):
468514
# test --list-tests
469515
tests = [self.create_test() for i in range(5)]
470516
output = self.run_tests('--list-tests', *tests)
471517
self.assertEqual(output.rstrip().splitlines(),
472518
tests)
473519

520+
def test_list_cases(self):
521+
# test --list-cases
522+
code = textwrap.dedent("""
523+
import unittest
524+
525+
class Tests(unittest.TestCase):
526+
def test_method1(self):
527+
pass
528+
def test_method2(self):
529+
pass
530+
""")
531+
testname = self.create_test(code=code)
532+
533+
# Test --list-cases
534+
all_methods = ['%s.Tests.test_method1' % testname,
535+
'%s.Tests.test_method2' % testname]
536+
output = self.run_tests('--list-cases', testname)
537+
self.assertEqual(output.splitlines(), all_methods)
538+
539+
# Test --list-cases with --match
540+
all_methods = ['%s.Tests.test_method1' % testname]
541+
output = self.run_tests('--list-cases',
542+
'-m', 'test_method1',
543+
testname)
544+
self.assertEqual(output.splitlines(), all_methods)
545+
474546
def test_crashed(self):
475547
# Any code which causes a crash
476548
code = 'import test.support; test.support._crash_python()'
477549
crash_test = self.create_test(name="crash", code=code)
478550
ok_test = self.create_test(name="ok")
479551

480552
tests = [crash_test, ok_test]
481-
output = self.run_tests("-j2", *tests, exitcode=1)
553+
output = self.run_tests("-j2", *tests, exitcode=2)
482554
self.check_executed_tests(output, tests, failed=crash_test,
483555
randomize=True)
484556

@@ -532,26 +604,28 @@ def test_main():
532604
subset = ['test_method1', 'test_method3']
533605
self.assertEqual(methods, subset)
534606

535-
def test_list_cases(self):
536-
# test --list-cases
607+
def test_env_changed(self):
537608
code = textwrap.dedent("""
538609
import unittest
539610
from test import support
540611
541612
class Tests(unittest.TestCase):
542-
def test_method1(self):
543-
pass
544-
def test_method2(self):
545-
pass
613+
def test_env_changed(self):
614+
open("env_changed", "w").close()
546615
547616
def test_main():
548617
support.run_unittest(Tests)
549618
""")
550619
testname = self.create_test(code=code)
551-
all_methods = ['%s.Tests.test_method1' % testname,
552-
'%s.Tests.test_method2' % testname]
553-
output = self.run_tests('--list-cases', testname)
554-
self.assertEqual(output.splitlines(), all_methods)
620+
621+
# don't fail by default
622+
output = self.run_tests(testname)
623+
self.check_executed_tests(output, [testname], env_changed=testname)
624+
625+
# fail with --fail-env-changed
626+
output = self.run_tests("--fail-env-changed", testname, exitcode=3)
627+
self.check_executed_tests(output, [testname], env_changed=testname,
628+
fail_env_changed=True)
555629

556630

557631
def test_main():

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.