Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit d4310fe

Browse filesBrowse files
avivkellermarco-ippolito
authored and
RafaelGSS
committed
test_runner: add support for coverage thresholds
Co-Authored-By: Marco Ippolito <marcoippolito54@gmail.com> PR-URL: #54429 Reviewed-By: Matteo Collina <matteo.collina@gmail.com> Reviewed-By: Paolo Insogna <paolo@cowtech.it> Reviewed-By: Moshe Atlow <moshe@atlow.co.il> Reviewed-By: Colin Ihrig <cjihrig@gmail.com> Reviewed-By: Marco Ippolito <marcoippolito54@gmail.com>
1 parent b5a23c9 commit d4310fe
Copy full SHA for d4310fe

File tree

Expand file treeCollapse file tree

7 files changed

+207
-0
lines changed
Open diff view settings
Filter options
Expand file treeCollapse file tree

7 files changed

+207
-0
lines changed
Open diff view settings
Collapse file

‎doc/api/cli.md‎

Copy file name to clipboardExpand all lines: doc/api/cli.md
+36Lines changed: 36 additions & 0 deletions
  • Display the source diff
  • Display the rich diff
Original file line numberDiff line numberDiff line change
@@ -2218,6 +2218,17 @@ concurrently. If `--experimental-test-isolation` is set to `'none'`, this flag
22182218
is ignored and concurrency is one. Otherwise, concurrency defaults to
22192219
`os.availableParallelism() - 1`.
22202220

2221+
### `--test-coverage-branches=threshold`
2222+
2223+
<!-- YAML
2224+
added: REPLACEME
2225+
-->
2226+
2227+
> Stability: 1 - Experimental
2228+
2229+
Require a minimum percent of covered branches. If code coverage does not reach
2230+
the threshold specified, the process will exit with code `1`.
2231+
22212232
### `--test-coverage-exclude`
22222233

22232234
<!-- YAML
@@ -2235,6 +2246,17 @@ This option may be specified multiple times to exclude multiple glob patterns.
22352246
If both `--test-coverage-exclude` and `--test-coverage-include` are provided,
22362247
files must meet **both** criteria to be included in the coverage report.
22372248

2249+
### `--test-coverage-functions=threshold`
2250+
2251+
<!-- YAML
2252+
added: REPLACEME
2253+
-->
2254+
2255+
> Stability: 1 - Experimental
2256+
2257+
Require a minimum percent of covered functions. If code coverage does not reach
2258+
the threshold specified, the process will exit with code `1`.
2259+
22382260
### `--test-coverage-include`
22392261

22402262
<!-- YAML
@@ -2252,6 +2274,17 @@ This option may be specified multiple times to include multiple glob patterns.
22522274
If both `--test-coverage-exclude` and `--test-coverage-include` are provided,
22532275
files must meet **both** criteria to be included in the coverage report.
22542276

2277+
### `--test-coverage-lines=threshold`
2278+
2279+
<!-- YAML
2280+
added: REPLACEME
2281+
-->
2282+
2283+
> Stability: 1 - Experimental
2284+
2285+
Require a minimum percent of covered lines. If code coverage does not reach
2286+
the threshold specified, the process will exit with code `1`.
2287+
22552288
### `--test-force-exit`
22562289

22572290
<!-- YAML
@@ -3047,8 +3080,11 @@ one is included in the list below.
30473080
* `--secure-heap-min`
30483081
* `--secure-heap`
30493082
* `--snapshot-blob`
3083+
* `--test-coverage-branches`
30503084
* `--test-coverage-exclude`
3085+
* `--test-coverage-functions`
30513086
* `--test-coverage-include`
3087+
* `--test-coverage-lines`
30523088
* `--test-name-pattern`
30533089
* `--test-only`
30543090
* `--test-reporter-destination`
Collapse file

‎doc/node.1‎

Copy file name to clipboardExpand all lines: doc/node.1
+9Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -450,12 +450,21 @@ Starts the Node.js command line test runner.
450450
The maximum number of test files that the test runner CLI will execute
451451
concurrently.
452452
.
453+
.It Fl -test-coverage-branches Ns = Ns Ar threshold
454+
Require a minimum threshold for branch coverage (0 - 100).
455+
.
453456
.It Fl -test-coverage-exclude
454457
A glob pattern that excludes matching files from the coverage report
455458
.
459+
.It Fl -test-coverage-functions Ns = Ns Ar threshold
460+
Require a minimum threshold for function coverage (0 - 100).
461+
.
456462
.It Fl -test-coverage-include
457463
A glob pattern that only includes matching files in the coverage report
458464
.
465+
.It Fl -test-coverage-lines Ns = Ns Ar threshold
466+
Require a minimum threshold for line coverage (0 - 100).
467+
.
459468
.It Fl -test-force-exit
460469
Configures the test runner to exit the process once all known tests have
461470
finished executing even if the event loop would otherwise remain active.
Collapse file

‎lib/internal/test_runner/test.js‎

Copy file name to clipboardExpand all lines: lib/internal/test_runner/test.js
+21Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ const {
1111
FunctionPrototype,
1212
MathMax,
1313
Number,
14+
NumberPrototypeToFixed,
1415
ObjectDefineProperty,
1516
ObjectSeal,
1617
PromisePrototypeThen,
@@ -28,6 +29,7 @@ const {
2829
SymbolDispose,
2930
} = primordials;
3031
const { getCallerLocation } = internalBinding('util');
32+
const { exitCodes: { kGenericUserError } } = internalBinding('errors');
3133
const { addAbortListener } = require('internal/events/abort_listener');
3234
const { queueMicrotask } = require('internal/process/task_queues');
3335
const { AsyncResource } = require('async_hooks');
@@ -1009,6 +1011,25 @@ class Test extends AsyncResource {
10091011

10101012
if (coverage) {
10111013
reporter.coverage(nesting, loc, coverage);
1014+
1015+
const coverages = [
1016+
{ __proto__: null, actual: coverage.totals.coveredLinePercent,
1017+
threshold: this.config.lineCoverage, name: 'line' },
1018+
1019+
{ __proto__: null, actual: coverage.totals.coveredBranchPercent,
1020+
threshold: this.config.branchCoverage, name: 'branch' },
1021+
1022+
{ __proto__: null, actual: coverage.totals.coveredFunctionPercent,
1023+
threshold: this.config.functionCoverage, name: 'function' },
1024+
];
1025+
1026+
for (let i = 0; i < coverages.length; i++) {
1027+
const { threshold, actual, name } = coverages[i];
1028+
if (actual < threshold) {
1029+
process.exitCode = kGenericUserError;
1030+
reporter.diagnostic(nesting, loc, `Error: ${NumberPrototypeToFixed(actual, 2)}% ${name} coverage does not meet threshold of ${threshold}%.`);
1031+
}
1032+
}
10121033
}
10131034

10141035
if (harness.watching) {
Collapse file

‎lib/internal/test_runner/utils.js‎

Copy file name to clipboardExpand all lines: lib/internal/test_runner/utils.js
+15Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ const {
3939
kIsNodeError,
4040
} = require('internal/errors');
4141
const { compose } = require('stream');
42+
const { validateInteger } = require('internal/validators');
4243

4344
const coverageColors = {
4445
__proto__: null,
@@ -194,6 +195,9 @@ function parseCommandLine() {
194195
let concurrency;
195196
let coverageExcludeGlobs;
196197
let coverageIncludeGlobs;
198+
let lineCoverage;
199+
let branchCoverage;
200+
let functionCoverage;
197201
let destinations;
198202
let isolation;
199203
let only = getOptionValue('--test-only');
@@ -278,6 +282,14 @@ function parseCommandLine() {
278282
if (coverage) {
279283
coverageExcludeGlobs = getOptionValue('--test-coverage-exclude');
280284
coverageIncludeGlobs = getOptionValue('--test-coverage-include');
285+
286+
branchCoverage = getOptionValue('--test-coverage-branches');
287+
lineCoverage = getOptionValue('--test-coverage-lines');
288+
functionCoverage = getOptionValue('--test-coverage-functions');
289+
290+
validateInteger(branchCoverage, '--test-coverage-branches', 0, 100);
291+
validateInteger(lineCoverage, '--test-coverage-lines', 0, 100);
292+
validateInteger(functionCoverage, '--test-coverage-functions', 0, 100);
281293
}
282294

283295
const setup = reporterScope.bind(async (rootReporter) => {
@@ -299,6 +311,9 @@ function parseCommandLine() {
299311
destinations,
300312
forceExit,
301313
isolation,
314+
branchCoverage,
315+
functionCoverage,
316+
lineCoverage,
302317
only,
303318
reporters,
304319
setup,
Collapse file

‎src/node_options.cc‎

Copy file name to clipboardExpand all lines: src/node_options.cc
+13Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -670,6 +670,19 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
670670
AddOption("--experimental-test-coverage",
671671
"enable code coverage in the test runner",
672672
&EnvironmentOptions::test_runner_coverage);
673+
AddOption("--test-coverage-branches",
674+
"the branch coverage minimum threshold",
675+
&EnvironmentOptions::test_coverage_branches,
676+
kAllowedInEnvvar);
677+
AddOption("--test-coverage-functions",
678+
"the function coverage minimum threshold",
679+
&EnvironmentOptions::test_coverage_functions,
680+
kAllowedInEnvvar);
681+
AddOption("--test-coverage-lines",
682+
"the line coverage minimum threshold",
683+
&EnvironmentOptions::test_coverage_lines,
684+
kAllowedInEnvvar);
685+
673686
AddOption("--experimental-test-isolation",
674687
"configures the type of test isolation used in the test runner",
675688
&EnvironmentOptions::test_isolation);
Collapse file

‎src/node_options.h‎

Copy file name to clipboardExpand all lines: src/node_options.h
+3Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,9 @@ class EnvironmentOptions : public Options {
184184
uint64_t test_runner_timeout = 0;
185185
bool test_runner_coverage = false;
186186
bool test_runner_force_exit = false;
187+
uint64_t test_coverage_branches = 0;
188+
uint64_t test_coverage_functions = 0;
189+
uint64_t test_coverage_lines = 0;
187190
bool test_runner_module_mocks = false;
188191
bool test_runner_snapshots = false;
189192
bool test_runner_update_snapshots = false;
Collapse file

‎test/parallel/test-runner-coverage-thresholds.js‎

Copy file name to clipboard
+110Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
'use strict';
2+
const common = require('../common');
3+
const assert = require('node:assert');
4+
const { spawnSync } = require('node:child_process');
5+
const { readdirSync } = require('node:fs');
6+
const { test } = require('node:test');
7+
const fixtures = require('../common/fixtures');
8+
const tmpdir = require('../common/tmpdir');
9+
10+
common.skipIfInspectorDisabled();
11+
tmpdir.refresh();
12+
13+
function findCoverageFileForPid(pid) {
14+
const pattern = `^coverage\\-${pid}\\-(\\d{13})\\-(\\d+)\\.json$`;
15+
const regex = new RegExp(pattern);
16+
17+
return readdirSync(tmpdir.path).find((file) => {
18+
return regex.test(file);
19+
});
20+
}
21+
22+
function getTapCoverageFixtureReport() {
23+
/* eslint-disable @stylistic/js/max-len */
24+
const report = [
25+
'# start of coverage report',
26+
'# -------------------------------------------------------------------------------------------------------------------',
27+
'# file | line % | branch % | funcs % | uncovered lines',
28+
'# -------------------------------------------------------------------------------------------------------------------',
29+
'# test/fixtures/test-runner/coverage.js | 78.65 | 38.46 | 60.00 | 12-13 16-22 27 39 43-44 61-62 66-67 71-72',
30+
'# test/fixtures/test-runner/invalid-tap.js | 100.00 | 100.00 | 100.00 | ',
31+
'# test/fixtures/v8-coverage/throw.js | 71.43 | 50.00 | 100.00 | 5-6',
32+
'# -------------------------------------------------------------------------------------------------------------------',
33+
'# all files | 78.35 | 43.75 | 60.00 |',
34+
'# -------------------------------------------------------------------------------------------------------------------',
35+
'# end of coverage report',
36+
].join('\n');
37+
/* eslint-enable @stylistic/js/max-len */
38+
39+
if (common.isWindows) {
40+
return report.replaceAll('/', '\\');
41+
}
42+
43+
return report;
44+
}
45+
46+
const fixture = fixtures.path('test-runner', 'coverage.js');
47+
const neededArguments = [
48+
'--experimental-test-coverage',
49+
'--test-reporter', 'tap',
50+
];
51+
52+
const coverages = [
53+
{ flag: '--test-coverage-lines', name: 'line', actual: 78.35 },
54+
{ flag: '--test-coverage-functions', name: 'function', actual: 60.00 },
55+
{ flag: '--test-coverage-branches', name: 'branch', actual: 43.75 },
56+
];
57+
58+
for (const coverage of coverages) {
59+
test(`test passing ${coverage.flag}`, async (t) => {
60+
const result = spawnSync(process.execPath, [
61+
...neededArguments,
62+
`${coverage.flag}=25`,
63+
fixture,
64+
]);
65+
66+
const stdout = result.stdout.toString();
67+
assert(stdout.includes(getTapCoverageFixtureReport()));
68+
assert.doesNotMatch(stdout, RegExp(`Error: [\\d\\.]+% ${coverage.name} coverage`));
69+
assert.strictEqual(result.status, 0);
70+
assert(!findCoverageFileForPid(result.pid));
71+
});
72+
73+
test(`test failing ${coverage.flag}`, async (t) => {
74+
const result = spawnSync(process.execPath, [
75+
...neededArguments,
76+
`${coverage.flag}=99`,
77+
fixture,
78+
]);
79+
80+
const stdout = result.stdout.toString();
81+
assert(stdout.includes(getTapCoverageFixtureReport()));
82+
assert.match(stdout, RegExp(`Error: ${coverage.actual.toFixed(2)}% ${coverage.name} coverage does not meet threshold of 99%`));
83+
assert.strictEqual(result.status, 1);
84+
assert(!findCoverageFileForPid(result.pid));
85+
});
86+
87+
test(`test out-of-range ${coverage.flag} (too high)`, async (t) => {
88+
const result = spawnSync(process.execPath, [
89+
...neededArguments,
90+
`${coverage.flag}=101`,
91+
fixture,
92+
]);
93+
94+
assert.match(result.stderr.toString(), RegExp(`The value of "${coverage.flag}`));
95+
assert.strictEqual(result.status, 1);
96+
assert(!findCoverageFileForPid(result.pid));
97+
});
98+
99+
test(`test out-of-range ${coverage.flag} (too low)`, async (t) => {
100+
const result = spawnSync(process.execPath, [
101+
...neededArguments,
102+
`${coverage.flag}=-1`,
103+
fixture,
104+
]);
105+
106+
assert.match(result.stderr.toString(), RegExp(`The value of "${coverage.flag}`));
107+
assert.strictEqual(result.status, 1);
108+
assert(!findCoverageFileForPid(result.pid));
109+
});
110+
}

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.