Skip to content

Commit 895db5a

Browse files
aduh95aduh95
committed
tools: add more options to track flaky tests
Refs: nodejs#43929 (comment)
1 parent 373bbce commit 895db5a

10 files changed

+39
-24
lines changed

‎.github/workflows/build-tarball.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ concurrency:
2929

3030
env:
3131
PYTHON_VERSION: '3.10'
32-
FLAKY_TESTS: dontcare
32+
FLAKY_TESTS: keep_retrying
3333

3434
permissions:
3535
contents: read
@@ -94,4 +94,4 @@ jobs:
9494
- name: Test
9595
run: |
9696
cd $TAR_DIR
97-
make run-ci -j2 V=1 TEST_CI_ARGS="-p dots"
97+
make run-ci -j2 V=1 TEST_CI_ARGS="-p dots --measure-flakiness 10"

‎.github/workflows/build-windows.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ concurrency:
2424

2525
env:
2626
PYTHON_VERSION: '3.10'
27-
FLAKY_TESTS: dontcare
27+
FLAKY_TESTS: keep_retrying
2828

2929
permissions:
3030
contents: read

‎.github/workflows/coverage-linux.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ concurrency:
2727

2828
env:
2929
PYTHON_VERSION: '3.10'
30-
FLAKY_TESTS: dontcare
30+
FLAKY_TESTS: keep_retrying
3131

3232
permissions:
3333
contents: read
@@ -53,7 +53,7 @@ jobs:
5353
# TODO(bcoe): fix the couple tests that fail with the inspector enabled.
5454
# The cause is most likely coverage's use of the inspector.
5555
- name: Test
56-
run: NODE_V8_COVERAGE=coverage/tmp make test-cov -j2 V=1 TEST_CI_ARGS="-p dots" || exit 0
56+
run: NODE_V8_COVERAGE=coverage/tmp make test-cov -j2 V=1 TEST_CI_ARGS="-p dots --measure-flakiness 10" || exit 0
5757
- name: Report JS
5858
run: npx c8 report --check-coverage
5959
env:

‎.github/workflows/coverage-windows.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ concurrency:
2929

3030
env:
3131
PYTHON_VERSION: '3.10'
32-
FLAKY_TESTS: dontcare
32+
FLAKY_TESTS: keep_retrying
3333

3434
permissions:
3535
contents: read

‎.github/workflows/doc.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -40,4 +40,4 @@ jobs:
4040
name: docs
4141
path: out/doc
4242
- name: Test
43-
run: NODE=$(command -v node) make test-doc-ci TEST_CI_ARGS="-p actions"
43+
run: NODE=$(command -v node) make test-doc-ci TEST_CI_ARGS="-p actions --measure-flakiness 10"

‎.github/workflows/test-asan.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ concurrency:
3131
env:
3232
ASAN_OPTIONS: intercept_tls_get_addr=0
3333
PYTHON_VERSION: '3.10'
34-
FLAKY_TESTS: dontcare
34+
FLAKY_TESTS: keep_retrying
3535

3636
permissions:
3737
contents: read
@@ -58,4 +58,4 @@ jobs:
5858
- name: Build
5959
run: make build-ci -j2 V=1
6060
- name: Test
61-
run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions -t 300"
61+
run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions -t 300 --measure-flakiness 10"

‎.github/workflows/test-internet.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ concurrency:
2222

2323
env:
2424
PYTHON_VERSION: '3.10'
25-
FLAKY_TESTS: dontcare
25+
FLAKY_TESTS: keep_retrying
2626

2727
permissions:
2828
contents: read

‎.github/workflows/test-linux.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ concurrency:
2424

2525
env:
2626
PYTHON_VERSION: '3.10'
27-
FLAKY_TESTS: dontcare
27+
FLAKY_TESTS: keep_retrying
2828

2929
permissions:
3030
contents: read
@@ -46,4 +46,4 @@ jobs:
4646
- name: Build
4747
run: make build-ci -j2 V=1 CONFIG_FLAGS="--error-on-warn"
4848
- name: Test
49-
run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions"
49+
run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions --measure-flakiness 10"

‎.github/workflows/test-macos.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ concurrency:
3030

3131
env:
3232
PYTHON_VERSION: '3.10'
33-
FLAKY_TESTS: dontcare
33+
FLAKY_TESTS: keep_retrying
3434

3535
permissions:
3636
contents: read
@@ -60,4 +60,4 @@ jobs:
6060
- name: Build
6161
run: make build-ci -j2 V=1 CONFIG_FLAGS="--error-on-warn"
6262
- name: Test
63-
run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions"
63+
run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions --measure-flakiness 10"

‎tools/test.py

+25-10
Original file line numberDiff line numberDiff line change
@@ -96,10 +96,11 @@ def get_module(name, path):
9696

9797
class ProgressIndicator(object):
9898

99-
def __init__(self, cases, flaky_tests_mode):
99+
def __init__(self, cases, flaky_tests_mode, measure_flakiness):
100100
self.cases = cases
101101
self.serial_id = 0
102102
self.flaky_tests_mode = flaky_tests_mode
103+
self.measure_flakiness = measure_flakiness
103104
self.parallel_queue = Queue(len(cases))
104105
self.sequential_queue = Queue(len(cases))
105106
for case in cases:
@@ -211,10 +212,20 @@ def RunSingle(self, parallel, thread_id):
211212
if output.UnexpectedOutput():
212213
if FLAKY in output.test.outcomes and self.flaky_tests_mode == DONTCARE:
213214
self.flaky_failed.append(output)
215+
elif FLAKY in output.test.outcomes and self.flaky_tests_mode == KEEP_RETRYING:
216+
for _ in range(99):
217+
if not case.Run().UnexpectedOutput():
218+
self.flaky_failed.append(output)
219+
break
220+
# If after 100 tries, the test is not passing, it's not flaky.
221+
self.failed.append(output)
214222
else:
215223
self.failed.append(output)
216224
if output.HasCrashed():
217225
self.crashed += 1
226+
if self.measure_flakiness:
227+
outputs = [case.Run() for _ in range(self.measure_flakiness)]
228+
print(f" failed {len([i for i in outputs if i.UnexpectedOutput()])} out of {self.measure_flakiness}")
218229
else:
219230
self.succeeded += 1
220231
self.remaining -= 1
@@ -436,8 +447,8 @@ def Done(self):
436447

437448
class CompactProgressIndicator(ProgressIndicator):
438449

439-
def __init__(self, cases, flaky_tests_mode, templates):
440-
super(CompactProgressIndicator, self).__init__(cases, flaky_tests_mode)
450+
def __init__(self, cases, flaky_tests_mode, measure_flakiness, templates):
451+
super(CompactProgressIndicator, self).__init__(cases, flaky_tests_mode, measure_flakiness)
441452
self.templates = templates
442453
self.last_status_length = 0
443454
self.start_time = time.time()
@@ -492,29 +503,29 @@ def PrintProgress(self, name):
492503

493504
class ColorProgressIndicator(CompactProgressIndicator):
494505

495-
def __init__(self, cases, flaky_tests_mode):
506+
def __init__(self, cases, flaky_tests_mode, measure_flakiness):
496507
templates = {
497508
'status_line': "[%(mins)02i:%(secs)02i|\033[34m%%%(remaining) 4d\033[0m|\033[32m+%(passed) 4d\033[0m|\033[31m-%(failed) 4d\033[0m]: %(test)s",
498509
'stdout': "\033[1m%s\033[0m",
499510
'stderr': "\033[31m%s\033[0m",
500511
}
501-
super(ColorProgressIndicator, self).__init__(cases, flaky_tests_mode, templates)
512+
super(ColorProgressIndicator, self).__init__(cases, flaky_tests_mode, measure_flakiness, templates)
502513

503514
def ClearLine(self, last_line_length):
504515
print("\033[1K\r", end='')
505516

506517

507518
class MonochromeProgressIndicator(CompactProgressIndicator):
508519

509-
def __init__(self, cases, flaky_tests_mode):
520+
def __init__(self, cases, flaky_tests_mode, measure_flakiness):
510521
templates = {
511522
'status_line': "[%(mins)02i:%(secs)02i|%%%(remaining) 4d|+%(passed) 4d|-%(failed) 4d]: %(test)s",
512523
'stdout': '%s',
513524
'stderr': '%s',
514525
'clear': lambda last_line_length: ("\r" + (" " * last_line_length) + "\r"),
515526
'max_length': 78
516527
}
517-
super(MonochromeProgressIndicator, self).__init__(cases, flaky_tests_mode, templates)
528+
super(MonochromeProgressIndicator, self).__init__(cases, flaky_tests_mode, measure_flakiness, templates)
518529

519530
def ClearLine(self, last_line_length):
520531
print(("\r" + (" " * last_line_length) + "\r"), end='')
@@ -948,8 +959,8 @@ def GetTimeout(self, mode, section=''):
948959
timeout = timeout * 6
949960
return timeout
950961

951-
def RunTestCases(cases_to_run, progress, tasks, flaky_tests_mode):
952-
progress = PROGRESS_INDICATORS[progress](cases_to_run, flaky_tests_mode)
962+
def RunTestCases(cases_to_run, progress, tasks, flaky_tests_mode, measure_flakiness):
963+
progress = PROGRESS_INDICATORS[progress](cases_to_run, flaky_tests_mode, measure_flakiness)
953964
return progress.Run(tasks)
954965

955966
# -------------------------------------------
@@ -967,6 +978,7 @@ def RunTestCases(cases_to_run, progress, tasks, flaky_tests_mode):
967978
SLOW = 'slow'
968979
FLAKY = 'flaky'
969980
DONTCARE = 'dontcare'
981+
KEEP_RETRYING = 'keep_retrying'
970982

971983
class Expression(object):
972984
pass
@@ -1357,6 +1369,9 @@ def BuildOptions():
13571369
result.add_option("--flaky-tests",
13581370
help="Regard tests marked as flaky (run|skip|dontcare)",
13591371
default="run")
1372+
result.add_option("--measure-flakiness",
1373+
help="When a test fails, re-run it x number of times",
1374+
default=0, type="int")
13601375
result.add_option("--skip-tests",
13611376
help="Tests that should not be executed (comma-separated)",
13621377
default="")
@@ -1733,7 +1748,7 @@ def should_keep(case):
17331748
else:
17341749
try:
17351750
start = time.time()
1736-
if RunTestCases(cases_to_run, options.progress, options.j, options.flaky_tests):
1751+
if RunTestCases(cases_to_run, options.progress, options.j, options.flaky_tests, options.measure_flakiness):
17371752
result = 0
17381753
else:
17391754
result = 1

0 commit comments

Comments
 (0)