diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 9871a28dbf2d2b..0d92eaf96ea033 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -240,9 +240,10 @@ def find_tests(self, tests): print("Couldn't find starting test (%s), using all tests" % self.ns.start, file=sys.stderr) + if self.ns.random_seed is None: + self.ns.random_seed = random.randrange(2 ** 32) + if self.ns.randomize: - if self.ns.random_seed is None: - self.ns.random_seed = random.randrange(10000000) random.seed(self.ns.random_seed) random.shuffle(self.selected) @@ -439,8 +440,8 @@ def run_tests(self): or self.tests or self.ns.args)): self.display_header() - if self.ns.randomize: - print("Using random seed", self.ns.random_seed) + if self.ns.random_seed is not None: + print("Random seed: {}".format(self.ns.random_seed)) if self.ns.forever: self.tests = self._test_forever(list(self.selected)) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index dbd463435c781b..f4cfcbe04400c6 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -2,6 +2,7 @@ import importlib import io import os +import random import sys import time import traceback @@ -161,6 +162,7 @@ def runtest_inner(ns, test, display_failure=True): with saved_test_environment(test, ns.verbose, ns.quiet, pgo=ns.pgo) as environment: start_time = time.time() the_module = importlib.import_module(abstest) + # If the test has a test_main, that will run the appropriate # tests. If not, use normal unittest test loading. test_runner = getattr(the_module, "test_main", None) @@ -173,9 +175,17 @@ def test_runner(): if loader.errors: raise Exception("errors while loading tests") support.run_unittest(tests) - test_runner() + + def final_test(): + if ns.random_seed is not None: + # bpo-31174: Reseed the RNG before each test file + # to get reproductible results + random.seed(ns.random_seed) + test_runner() + + final_test() if ns.huntrleaks: - refleak = dash_R(the_module, test, test_runner, ns.huntrleaks) + refleak = dash_R(the_module, test, final_test, ns.huntrleaks) test_time = time.time() - start_time post_test_cleanup() except support.ResourceDenied as msg: diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index b756839748a158..a99f53268ab941 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -457,9 +457,9 @@ def list_regex(line_format, tests): self.check_line(output, 'Tests result: %s' % result) def parse_random_seed(self, output): - match = self.regex_search(r'Using random seed ([0-9]+)', output) + match = self.regex_search(r'Random seed: ([0-9]+)', output) randseed = int(match.group(1)) - self.assertTrue(0 <= randseed <= 10000000, randseed) + self.assertTrue(0 <= randseed < 2 ** 32, randseed) return randseed def run_command(self, args, input=None, exitcode=0, **kw): @@ -950,6 +950,41 @@ def test_env_changed(self): self.check_executed_tests(output, [testname], env_changed=testname, fail_env_changed=True) + def check_random_reseed(self, parallel): + # bpo-31174: Each test file should be run with the same random seed + code = textwrap.dedent(""" + import random + import unittest + + class Tests(unittest.TestCase): + def test_random(self): + print("Rand1000: %s" % random.randint(0, 1000)) + """) + testname = self.create_test(code=code) + + tests = [testname] * 3 + if parallel: + output = self.run_tests("-j3", *tests) + else: + output = self.run_tests(*tests) + self.check_executed_tests(output, tests) + + # Get random numbers + numbers = (line for line in output.splitlines() + if line.startswith("Rand1000:")) + numbers = (line[9:].strip() for line in numbers) + numbers = map(int, numbers) + + # All "random" numbers must be the same + numbers = set(numbers) + self.assertEqual(len(numbers), 1, numbers) + + def test_random_reseed_sequential(self): + self.check_random_reseed(False) + + def test_random_reseed_parallel(self): + self.check_random_reseed(True) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Tests/2017-08-10-14-11-09.bpo-31174.ARF9nb.rst b/Misc/NEWS.d/next/Tests/2017-08-10-14-11-09.bpo-31174.ARF9nb.rst new file mode 100644 index 00000000000000..794758a04c9b78 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2017-08-10-14-11-09.bpo-31174.ARF9nb.rst @@ -0,0 +1,2 @@ +regrtest now reseeds the random RNG before each test file. Use also more +entropy for the seed: 2**32 (32 bits) rather than 10_000_000 (24 bits).