Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit ff1fcab

Browse filesBrowse files
JakobJingleheimervespa7
authored and
aduh95
committed
test_runner: support expecting a test-case to fail
Co-Authored-By: Alejandro Espa <98526766+vespa7@users.noreply.github.com> PR-URL: #60669 Reviewed-By: Aviv Keller <me@aviv.sh> Reviewed-By: Pietro Marchini <pietro.marchini94@gmail.com> Reviewed-By: Moshe Atlow <moshe@atlow.co.il> Reviewed-By: Jordan Harband <ljharb@gmail.com>
1 parent 3f17acf commit ff1fcab
Copy full SHA for ff1fcab

22 files changed

+806-493Lines changed: 806 additions & 493 deletions
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎doc/api/test.md‎

Copy file name to clipboardExpand all lines: doc/api/test.md
+51Lines changed: 51 additions & 0 deletions
  • Display the source diff
  • Display the rich diff
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,57 @@ test('todo() method with message', (t) => {
224224
});
225225
```
226226

227+
## Expecting tests to fail
228+
229+
<!-- YAML
230+
added:
231+
- REPLACEME
232+
-->
233+
234+
This flips the pass/fail reporting for a specific test or suite: A flagged test/test-case must throw
235+
in order to "pass"; a test/test-case that does not throw, fails.
236+
237+
In the following, `doTheThing()` returns _currently_ `false` (`false` does not equal `true`, causing
238+
`strictEqual` to throw, so the test-case passes).
239+
240+
```js
241+
it.expectFailure('should do the thing', () => {
242+
assert.strictEqual(doTheThing(), true);
243+
});
244+
245+
it('should do the thing', { expectFailure: true }, () => {
246+
assert.strictEqual(doTheThing(), true);
247+
});
248+
```
249+
250+
`skip` and/or `todo` are mutually exclusive to `expectFailure`, and `skip` or `todo`
251+
will "win" when both are applied (`skip` wins against both, and `todo` wins
252+
against `expectFailure`).
253+
254+
These tests will be skipped (and not run):
255+
256+
```js
257+
it.expectFailure('should do the thing', { skip: true }, () => {
258+
assert.strictEqual(doTheThing(), true);
259+
});
260+
261+
it.skip('should do the thing', { expectFailure: true }, () => {
262+
assert.strictEqual(doTheThing(), true);
263+
});
264+
```
265+
266+
These tests will be marked "todo" (silencing errors):
267+
268+
```js
269+
it.expectFailure('should do the thing', { todo: true }, () => {
270+
assert.strictEqual(doTheThing(), true);
271+
});
272+
273+
it.todo('should do the thing', { expectFailure: true }, () => {
274+
assert.strictEqual(doTheThing(), true);
275+
});
276+
```
277+
227278
## `describe()` and `it()` aliases
228279

229280
Suites and tests can also be written using the `describe()` and `it()`
Collapse file

‎lib/internal/test_runner/harness.js‎

Copy file name to clipboardExpand all lines: lib/internal/test_runner/harness.js
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -377,7 +377,7 @@ function runInParentContext(Factory) {
377377

378378
return run(name, options, fn, overrides);
379379
};
380-
ArrayPrototypeForEach(['skip', 'todo', 'only'], (keyword) => {
380+
ArrayPrototypeForEach(['expectFailure', 'skip', 'todo', 'only'], (keyword) => {
381381
test[keyword] = (name, options, fn) => {
382382
const overrides = {
383383
__proto__: null,
Collapse file

‎lib/internal/test_runner/reporter/tap.js‎

Copy file name to clipboardExpand all lines: lib/internal/test_runner/reporter/tap.js
+5-3Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,12 @@ async function * tapReporter(source) {
3333
for await (const { type, data } of source) {
3434
switch (type) {
3535
case 'test:fail': {
36-
yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo);
36+
yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo, data.expectFailure);
3737
const location = data.file ? `${data.file}:${data.line}:${data.column}` : null;
3838
yield reportDetails(data.nesting, data.details, location);
3939
break;
4040
} case 'test:pass':
41-
yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo);
41+
yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo, data.expectFailure);
4242
yield reportDetails(data.nesting, data.details, null);
4343
break;
4444
case 'test:plan':
@@ -65,7 +65,7 @@ async function * tapReporter(source) {
6565
}
6666
}
6767

68-
function reportTest(nesting, testNumber, status, name, skip, todo) {
68+
function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure) {
6969
let line = `${indent(nesting)}${status} ${testNumber}`;
7070

7171
if (name) {
@@ -76,6 +76,8 @@ function reportTest(nesting, testNumber, status, name, skip, todo) {
7676
line += ` # SKIP${typeof skip === 'string' && skip.length ? ` ${tapEscape(skip)}` : ''}`;
7777
} else if (todo !== undefined) {
7878
line += ` # TODO${typeof todo === 'string' && todo.length ? ` ${tapEscape(todo)}` : ''}`;
79+
} else if (expectFailure !== undefined) {
80+
line += ' # EXPECTED FAILURE';
7981
}
8082

8183
line += '\n';
Collapse file

‎lib/internal/test_runner/reporter/utils.js‎

Copy file name to clipboardExpand all lines: lib/internal/test_runner/reporter/utils.js
+3-1Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,14 +71,16 @@ function formatError(error, indent) {
7171
function formatTestReport(type, data, prefix = '', indent = '', hasChildren = false, showErrorDetails = true) {
7272
let color = reporterColorMap[type] ?? colors.white;
7373
let symbol = reporterUnicodeSymbolMap[type] ?? ' ';
74-
const { skip, todo } = data;
74+
const { skip, todo, expectFailure } = data;
7575
const duration_ms = data.details?.duration_ms ? ` ${colors.gray}(${data.details.duration_ms}ms)${colors.white}` : '';
7676
let title = `${data.name}${duration_ms}`;
7777

7878
if (skip !== undefined) {
7979
title += ` # ${typeof skip === 'string' && skip.length ? skip : 'SKIP'}`;
8080
} else if (todo !== undefined) {
8181
title += ` # ${typeof todo === 'string' && todo.length ? todo : 'TODO'}`;
82+
} else if (expectFailure !== undefined) {
83+
title += ` # EXPECTED FAILURE`;
8284
}
8385

8486
const error = showErrorDetails ? formatError(data.details?.error, indent) : '';
Collapse file

‎lib/internal/test_runner/test.js‎

Copy file name to clipboardExpand all lines: lib/internal/test_runner/test.js
+11-2Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -496,7 +496,7 @@ class Test extends AsyncResource {
496496
super('Test');
497497

498498
let { fn, name, parent } = options;
499-
const { concurrency, entryFile, loc, only, timeout, todo, skip, signal, plan } = options;
499+
const { concurrency, entryFile, expectFailure, loc, only, timeout, todo, skip, signal, plan } = options;
500500

501501
if (typeof fn !== 'function') {
502502
fn = noop;
@@ -635,6 +635,7 @@ class Test extends AsyncResource {
635635
this.plan = null;
636636
this.expectedAssertions = plan;
637637
this.cancelled = false;
638+
this.expectFailure = expectFailure !== undefined && expectFailure !== false;
638639
this.skipped = skip !== undefined && skip !== false;
639640
this.isTodo = (todo !== undefined && todo !== false) || this.parent?.isTodo;
640641
this.startTime = null;
@@ -938,7 +939,12 @@ class Test extends AsyncResource {
938939
return;
939940
}
940941

941-
this.passed = false;
942+
if (this.expectFailure === true) {
943+
this.passed = true;
944+
} else {
945+
this.passed = false;
946+
}
947+
942948
this.error = err;
943949
}
944950

@@ -1335,6 +1341,8 @@ class Test extends AsyncResource {
13351341
directive = this.reporter.getSkip(this.message);
13361342
} else if (this.isTodo) {
13371343
directive = this.reporter.getTodo(this.message);
1344+
} else if (this.expectFailure) {
1345+
directive = this.reporter.getXFail(this.expectFailure); // TODO(@JakobJingleheimer): support specifying failure
13381346
}
13391347

13401348
if (this.reportedType) {
@@ -1349,6 +1357,7 @@ class Test extends AsyncResource {
13491357
if (this.passedAttempt !== undefined) {
13501358
details.passed_on_attempt = this.passedAttempt;
13511359
}
1360+
13521361
return { __proto__: null, details, directive };
13531362
}
13541363

Collapse file

‎lib/internal/test_runner/tests_stream.js‎

Copy file name to clipboardExpand all lines: lib/internal/test_runner/tests_stream.js
+4Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,10 @@ class TestsStream extends Readable {
8787
return { __proto__: null, todo: reason ?? true };
8888
}
8989

90+
getXFail(expectation = undefined) {
91+
return { __proto__: null, expectFailure: expectation ?? true };
92+
}
93+
9094
enqueue(nesting, loc, name, type) {
9195
this[kEmitMessage]('test:enqueue', {
9296
__proto__: null,
Collapse file

‎test/fixtures/test-runner/output/describe_it.js‎

Copy file name to clipboardExpand all lines: test/fixtures/test-runner/output/describe_it.js
+24-1Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,23 @@ const { describe, it, test } = require('node:test');
55
const util = require('util');
66

77

8-
it.todo('sync pass todo', () => {
8+
it.expectFailure('sync expect fail (method)', () => {
9+
throw new Error('should pass');
10+
});
11+
12+
it('sync expect fail (options)', { expectFailure: true }, () => {
13+
throw new Error('should pass');
14+
});
915

16+
it.expectFailure('async expect fail (method)', async () => {
17+
throw new Error('should pass');
18+
});
19+
20+
it('async expect fail (options)', { expectFailure: true }, async () => {
21+
throw new Error('should pass');
22+
});
23+
24+
it.todo('sync pass todo', () => {
1025
});
1126

1227
it('sync pass todo with message', { todo: 'this is a passing todo' }, () => {
@@ -16,13 +31,21 @@ it.todo('sync todo', () => {
1631
throw new Error('should not count as a failure');
1732
});
1833

34+
it.todo('sync todo with expect fail', { expectFailure: true }, () => {
35+
throw new Error('should not count as an expected failure');
36+
});
37+
1938
it('sync todo with message', { todo: 'this is a failing todo' }, () => {
2039
throw new Error('should not count as a failure');
2140
});
2241

2342
it.skip('sync skip pass', () => {
2443
});
2544

45+
it.skip('sync skip expect fail', { expectFailure: true }, () => {
46+
throw new Error('should not fail');
47+
});
48+
2649
it('sync skip pass with message', { skip: 'this is skipped' }, () => {
2750
});
2851

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.