Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit a1d5209

Browse filesBrowse files
aduh95danielleadams
aduh95
authored and
danielleadams
committed
tools: add more options to track flaky tests
Refs: #43929 (comment) PR-URL: #43954 Backport-PR-URL: #45126 Reviewed-By: Matteo Collina <matteo.collina@gmail.com> Reviewed-By: Tobias Nießen <tniessen@tnie.de> Reviewed-By: Feng Yu <F3n67u@outlook.com>
1 parent 98c49d8 commit a1d5209
Copy full SHA for a1d5209

File tree

Expand file treeCollapse file tree

7 files changed

+40
-24
lines changed
Open diff view settings
Filter options
Expand file treeCollapse file tree

7 files changed

+40
-24
lines changed
Open diff view settings
Collapse file

‎.github/workflows/build-tarball.yml‎

Copy file name to clipboardExpand all lines: .github/workflows/build-tarball.yml
+3-4Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,11 @@ on:
1111
- v[0-9]+.x
1212

1313
env:
14-
FLAKY_TESTS: dontcare
14+
PYTHON_VERSION: '3.10'
15+
FLAKY_TESTS: keep_retrying
1516

1617
jobs:
1718
build-tarball:
18-
env:
19-
PYTHON_VERSION: '3.10'
2019
runs-on: ubuntu-latest
2120
steps:
2221
- uses: actions/checkout@v2
@@ -71,4 +70,4 @@ jobs:
7170
- name: Test
7271
run: |
7372
cd $TAR_DIR
74-
make run-ci -j2 V=1 TEST_CI_ARGS="-p dots"
73+
make run-ci -j2 V=1 TEST_CI_ARGS="-p dots --measure-flakiness 9"
Collapse file

‎.github/workflows/build-windows.yml‎

Copy file name to clipboardExpand all lines: .github/workflows/build-windows.yml
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ on:
1313

1414
env:
1515
PYTHON_VERSION: '3.10'
16-
FLAKY_TESTS: dontcare
16+
FLAKY_TESTS: keep_retrying
1717

1818
jobs:
1919
build-windows:
Collapse file

‎.github/workflows/misc.yml‎

Copy file name to clipboardExpand all lines: .github/workflows/misc.yml
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,4 @@ jobs:
3131
name: docs
3232
path: out/doc
3333
- name: Test
34-
run: NODE=$(command -v node) make test-doc-ci TEST_CI_ARGS="-p actions"
34+
run: NODE=$(command -v node) make test-doc-ci TEST_CI_ARGS="-p actions --measure-flakiness 9"
Collapse file

‎.github/workflows/test-asan.yml‎

Copy file name to clipboardExpand all lines: .github/workflows/test-asan.yml
+2-2Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ on:
1919

2020
env:
2121
PYTHON_VERSION: '3.10'
22-
FLAKY_TESTS: dontcare
22+
FLAKY_TESTS: keep_retrying
2323

2424
jobs:
2525
test-asan:
@@ -40,4 +40,4 @@ jobs:
4040
- name: Build
4141
run: make build-ci -j2 V=1
4242
- name: Test
43-
run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions -t 300"
43+
run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions -t 300 --measure-flakiness 9"
Collapse file

‎.github/workflows/test-linux.yml‎

Copy file name to clipboardExpand all lines: .github/workflows/test-linux.yml
+2-2Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ on:
1313

1414
env:
1515
PYTHON_VERSION: '3.10'
16-
FLAKY_TESTS: dontcare
16+
FLAKY_TESTS: keep_retrying
1717

1818
jobs:
1919
test-linux:
@@ -29,4 +29,4 @@ jobs:
2929
- name: Build
3030
run: make build-ci -j2 V=1 CONFIG_FLAGS="--error-on-warn"
3131
- name: Test
32-
run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions"
32+
run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions --measure-flakiness 9"
Collapse file

‎.github/workflows/test-macos.yml‎

Copy file name to clipboardExpand all lines: .github/workflows/test-macos.yml
+2-2Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ on:
1919

2020
env:
2121
PYTHON_VERSION: '3.10'
22-
FLAKY_TESTS: dontcare
22+
FLAKY_TESTS: keep_retrying
2323

2424
jobs:
2525
test-macOS:
@@ -35,4 +35,4 @@ jobs:
3535
- name: Build
3636
run: make build-ci -j2 V=1 CONFIG_FLAGS="--error-on-warn"
3737
- name: Test
38-
run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions"
38+
run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions --measure-flakiness 9"
Collapse file

‎tools/test.py‎

Copy file name to clipboardExpand all lines: tools/test.py
+29-12Lines changed: 29 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -96,10 +96,11 @@ def get_module(name, path):
9696

9797
class ProgressIndicator(object):
9898

99-
def __init__(self, cases, flaky_tests_mode):
99+
def __init__(self, cases, flaky_tests_mode, measure_flakiness):
100100
self.cases = cases
101101
self.serial_id = 0
102102
self.flaky_tests_mode = flaky_tests_mode
103+
self.measure_flakiness = measure_flakiness
103104
self.parallel_queue = Queue(len(cases))
104105
self.sequential_queue = Queue(len(cases))
105106
for case in cases:
@@ -211,10 +212,22 @@ def RunSingle(self, parallel, thread_id):
211212
if output.UnexpectedOutput():
212213
if FLAKY in output.test.outcomes and self.flaky_tests_mode == DONTCARE:
213214
self.flaky_failed.append(output)
215+
elif FLAKY in output.test.outcomes and self.flaky_tests_mode == KEEP_RETRYING:
216+
for _ in range(99):
217+
if not case.Run().UnexpectedOutput():
218+
self.flaky_failed.append(output)
219+
break
220+
else:
221+
# If after 100 tries, the test is not passing, it's not flaky.
222+
self.failed.append(output)
214223
else:
215224
self.failed.append(output)
216225
if output.HasCrashed():
217226
self.crashed += 1
227+
if self.measure_flakiness:
228+
outputs = [case.Run() for _ in range(self.measure_flakiness)]
229+
# +1s are there because the test already failed once at this point.
230+
print(" failed {} out of {}".format(len([i for i in outputs if i.UnexpectedOutput()]) + 1, self.measure_flakiness + 1))
218231
else:
219232
self.succeeded += 1
220233
self.remaining -= 1
@@ -436,8 +449,8 @@ def Done(self):
436449

437450
class CompactProgressIndicator(ProgressIndicator):
438451

439-
def __init__(self, cases, flaky_tests_mode, templates):
440-
super(CompactProgressIndicator, self).__init__(cases, flaky_tests_mode)
452+
def __init__(self, cases, flaky_tests_mode, measure_flakiness, templates):
453+
super(CompactProgressIndicator, self).__init__(cases, flaky_tests_mode, measure_flakiness)
441454
self.templates = templates
442455
self.last_status_length = 0
443456
self.start_time = time.time()
@@ -492,29 +505,29 @@ def PrintProgress(self, name):
492505

493506
class ColorProgressIndicator(CompactProgressIndicator):
494507

495-
def __init__(self, cases, flaky_tests_mode):
508+
def __init__(self, cases, flaky_tests_mode, measure_flakiness):
496509
templates = {
497510
'status_line': "[%(mins)02i:%(secs)02i|\033[34m%%%(remaining) 4d\033[0m|\033[32m+%(passed) 4d\033[0m|\033[31m-%(failed) 4d\033[0m]: %(test)s",
498511
'stdout': "\033[1m%s\033[0m",
499512
'stderr': "\033[31m%s\033[0m",
500513
}
501-
super(ColorProgressIndicator, self).__init__(cases, flaky_tests_mode, templates)
514+
super(ColorProgressIndicator, self).__init__(cases, flaky_tests_mode, measure_flakiness, templates)
502515

503516
def ClearLine(self, last_line_length):
504517
print("\033[1K\r", end='')
505518

506519

507520
class MonochromeProgressIndicator(CompactProgressIndicator):
508521

509-
def __init__(self, cases, flaky_tests_mode):
522+
def __init__(self, cases, flaky_tests_mode, measure_flakiness):
510523
templates = {
511524
'status_line': "[%(mins)02i:%(secs)02i|%%%(remaining) 4d|+%(passed) 4d|-%(failed) 4d]: %(test)s",
512525
'stdout': '%s',
513526
'stderr': '%s',
514527
'clear': lambda last_line_length: ("\r" + (" " * last_line_length) + "\r"),
515528
'max_length': 78
516529
}
517-
super(MonochromeProgressIndicator, self).__init__(cases, flaky_tests_mode, templates)
530+
super(MonochromeProgressIndicator, self).__init__(cases, flaky_tests_mode, measure_flakiness, templates)
518531

519532
def ClearLine(self, last_line_length):
520533
print(("\r" + (" " * last_line_length) + "\r"), end='')
@@ -946,8 +959,8 @@ def GetVm(self, arch, mode):
946959
def GetTimeout(self, mode):
947960
return self.timeout * TIMEOUT_SCALEFACTOR[ARCH_GUESS or 'ia32'][mode]
948961

949-
def RunTestCases(cases_to_run, progress, tasks, flaky_tests_mode):
950-
progress = PROGRESS_INDICATORS[progress](cases_to_run, flaky_tests_mode)
962+
def RunTestCases(cases_to_run, progress, tasks, flaky_tests_mode, measure_flakiness):
963+
progress = PROGRESS_INDICATORS[progress](cases_to_run, flaky_tests_mode, measure_flakiness)
951964
return progress.Run(tasks)
952965

953966
# -------------------------------------------
@@ -965,6 +978,7 @@ def RunTestCases(cases_to_run, progress, tasks, flaky_tests_mode):
965978
SLOW = 'slow'
966979
FLAKY = 'flaky'
967980
DONTCARE = 'dontcare'
981+
KEEP_RETRYING = 'keep_retrying'
968982

969983
class Expression(object):
970984
pass
@@ -1353,8 +1367,11 @@ def BuildOptions():
13531367
result.add_option("--cat", help="Print the source of the tests",
13541368
default=False, action="store_true")
13551369
result.add_option("--flaky-tests",
1356-
help="Regard tests marked as flaky (run|skip|dontcare)",
1370+
help="Regard tests marked as flaky (run|skip|dontcare|keep_retrying)",
13571371
default="run")
1372+
result.add_option("--measure-flakiness",
1373+
help="When a test fails, re-run it x number of times",
1374+
default=0, type="int")
13581375
result.add_option("--skip-tests",
13591376
help="Tests that should not be executed (comma-separated)",
13601377
default="")
@@ -1426,7 +1443,7 @@ def ProcessOptions(options):
14261443
# tends to exaggerate the number of available cpus/cores.
14271444
cores = os.environ.get('JOBS')
14281445
options.j = int(cores) if cores is not None else multiprocessing.cpu_count()
1429-
if options.flaky_tests not in [RUN, SKIP, DONTCARE]:
1446+
if options.flaky_tests not in [RUN, SKIP, DONTCARE, KEEP_RETRYING]:
14301447
print("Unknown flaky-tests mode %s" % options.flaky_tests)
14311448
return False
14321449
return True
@@ -1726,7 +1743,7 @@ def should_keep(case):
17261743
else:
17271744
try:
17281745
start = time.time()
1729-
if RunTestCases(cases_to_run, options.progress, options.j, options.flaky_tests):
1746+
if RunTestCases(cases_to_run, options.progress, options.j, options.flaky_tests, options.measure_flakiness):
17301747
result = 0
17311748
else:
17321749
result = 1

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.