Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 11f74e7

Browse filesBrowse files
panvaruyadorno
panva
authored and
ruyadorno
committed
test: update WPT runner
PR-URL: #43455 Reviewed-By: Rich Trott <rtrott@gmail.com> Reviewed-By: Joyee Cheung <joyeec9h3@gmail.com>
1 parent d57178c commit 11f74e7
Copy full SHA for 11f74e7

File tree

Expand file treeCollapse file tree

17 files changed

+3606
-136
lines changed
Open diff view settings
Filter options
Expand file treeCollapse file tree

17 files changed

+3606
-136
lines changed
Open diff view settings
Collapse file

‎test/common/wpt.js‎

Copy file name to clipboardExpand all lines: test/common/wpt.js
+66-14Lines changed: 66 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ const fixtures = require('../common/fixtures');
55
const fs = require('fs');
66
const fsPromises = fs.promises;
77
const path = require('path');
8+
const events = require('events');
89
const { inspect } = require('util');
910
const { Worker } = require('worker_threads');
1011

@@ -152,21 +153,30 @@ class WPTTestSpec {
152153
this.filename = filename;
153154

154155
this.requires = new Set();
155-
this.failReasons = [];
156+
this.failedTests = [];
157+
this.flakyTests = [];
156158
this.skipReasons = [];
157159
for (const item of rules) {
158160
if (item.requires.length) {
159161
for (const req of item.requires) {
160162
this.requires.add(req);
161163
}
162164
}
163-
if (item.fail) {
164-
this.failReasons.push(item.fail);
165+
if (Array.isArray(item.fail?.expected)) {
166+
this.failedTests.push(...item.fail.expected);
167+
}
168+
if (Array.isArray(item.fail?.flaky)) {
169+
this.failedTests.push(...item.fail.flaky);
170+
this.flakyTests.push(...item.fail.flaky);
165171
}
166172
if (item.skip) {
167173
this.skipReasons.push(item.skip);
168174
}
169175
}
176+
177+
this.failedTests = [...new Set(this.failedTests)];
178+
this.flakyTests = [...new Set(this.flakyTests)];
179+
this.skipReasons = [...new Set(this.skipReasons)];
170180
}
171181

172182
getRelativePath() {
@@ -368,7 +378,7 @@ class WPTRunner {
368378

369379
// TODO(joyeecheung): work with the upstream to port more tests in .html
370380
// to .js.
371-
runJsTests() {
381+
async runJsTests() {
372382
let queue = [];
373383

374384
// If the tests are run as `node test/wpt/test-something.js subset.any.js`,
@@ -459,6 +469,8 @@ class WPTRunner {
459469
);
460470
this.inProgress.delete(testFileName);
461471
});
472+
473+
await events.once(worker, 'exit').catch(() => {});
462474
}
463475

464476
process.on('exit', () => {
@@ -469,34 +481,72 @@ class WPTRunner {
469481
}
470482
}
471483
inspect.defaultOptions.depth = Infinity;
472-
console.log(this.results);
484+
// Sorts the rules to have consistent output
485+
console.log(JSON.stringify(Object.keys(this.results).sort().reduce(
486+
(obj, key) => {
487+
obj[key] = this.results[key];
488+
return obj;
489+
},
490+
{}
491+
), null, 2));
473492

474493
const failures = [];
475494
let expectedFailures = 0;
476495
let skipped = 0;
477-
for (const key of Object.keys(this.results)) {
478-
const item = this.results[key];
479-
if (item.fail && item.fail.unexpected) {
496+
for (const [key, item] of Object.entries(this.results)) {
497+
if (item.fail?.unexpected) {
480498
failures.push(key);
481499
}
482-
if (item.fail && item.fail.expected) {
500+
if (item.fail?.expected) {
483501
expectedFailures++;
484502
}
485503
if (item.skip) {
486504
skipped++;
487505
}
488506
}
507+
508+
const unexpectedPasses = [];
509+
for (const [key, specMap] of this.specMap) {
510+
// File has no expected failures
511+
if (!specMap.failedTests.length) {
512+
continue;
513+
}
514+
515+
// File was (maybe even conditionally) skipped
516+
if (this.results[key]?.skip) {
517+
continue;
518+
}
519+
520+
// Full check: every expected to fail test is present
521+
if (specMap.failedTests.some((expectedToFail) => {
522+
if (specMap.flakyTests.includes(expectedToFail)) {
523+
return false;
524+
}
525+
return this.results[key]?.fail?.expected?.includes(expectedToFail) !== true;
526+
})) {
527+
unexpectedPasses.push(key);
528+
continue;
529+
}
530+
}
531+
489532
const ran = total - skipped;
490533
const passed = ran - expectedFailures - failures.length;
491534
console.log(`Ran ${ran}/${total} tests, ${skipped} skipped,`,
492535
`${passed} passed, ${expectedFailures} expected failures,`,
493-
`${failures.length} unexpected failures`);
536+
`${failures.length} unexpected failures,`,
537+
`${unexpectedPasses.length} unexpected passes`);
494538
if (failures.length > 0) {
495539
const file = path.join('test', 'wpt', 'status', `${this.path}.json`);
496540
throw new Error(
497541
`Found ${failures.length} unexpected failures. ` +
498542
`Consider updating ${file} for these files:\n${failures.join('\n')}`);
499543
}
544+
if (unexpectedPasses.length > 0) {
545+
const file = path.join('test', 'wpt', 'status', `${this.path}.json`);
546+
throw new Error(
547+
`Found ${unexpectedPasses.length} unexpected passes. ` +
548+
`Consider updating ${file} for these files:\n${unexpectedPasses.join('\n')}`);
549+
}
500550
});
501551
}
502552

@@ -577,8 +627,9 @@ class WPTRunner {
577627
if (!result[item.status][key]) {
578628
result[item.status][key] = [];
579629
}
580-
if (result[item.status][key].indexOf(item.reason) === -1) {
581-
result[item.status][key].push(item.reason);
630+
const hasName = result[item.status][key].includes(item.name);
631+
if (!hasName) {
632+
result[item.status][key].push(item.name);
582633
}
583634
}
584635
}
@@ -589,10 +640,10 @@ class WPTRunner {
589640

590641
fail(filename, test, status) {
591642
const spec = this.specMap.get(filename);
592-
const expected = !!(spec.failReasons.length);
643+
const expected = spec.failedTests.includes(test.name);
593644
if (expected) {
594645
console.log(`[EXPECTED_FAILURE][${status.toUpperCase()}] ${test.name}`);
595-
console.log(spec.failReasons.join('; '));
646+
console.log(test.message || status);
596647
} else {
597648
console.log(`[UNEXPECTED_FAILURE][${status.toUpperCase()}] ${test.name}`);
598649
}
@@ -604,6 +655,7 @@ class WPTRunner {
604655
` ${require.main.filename} ${filename}`;
605656
console.log(`Command: ${command}\n`);
606657
this.addTestResult(filename, {
658+
name: test.name,
607659
expected,
608660
status: kFail,
609661
reason: test.message || status
Collapse file

‎test/wpt/README.md‎

Copy file name to clipboardExpand all lines: test/wpt/README.md
+16-3Lines changed: 16 additions & 3 deletions
  • Display the source diff
  • Display the rich diff
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,11 @@ add this to `test/wpt/status/url.json`:
9191

9292
```json
9393
"url-searchparams.any.js": {
94-
"fail": "explain why the test fails, ideally with links"
94+
"fail": {
95+
"expected": [
96+
"test name in the WPT test case, e.g. second argument passed to test()"
97+
]
98+
}
9599
}
96100
```
97101

@@ -155,8 +159,17 @@ expected failures.
155159
// Optional: the test will be skipped with the reason printed
156160
"skip": "explain why we cannot run a test that's supposed to pass",
157161
158-
// Optional: the test will be skipped with the reason printed
159-
"fail": "explain why we the test is expected to fail"
162+
// Optional: failing tests
163+
"fail": {
164+
"note": "You may leave an optional arbitrary note e.g. with TODOs",
165+
"expected": [
166+
"test name in the WPT test case, e.g. second argument passed to test()",
167+
"another test name"
168+
],
169+
"flaky": [
170+
"flaky test name"
171+
]
172+
}
160173
}
161174
}
162175
```
Collapse file

‎test/wpt/status/FileAPI/blob.json‎

Copy file name to clipboard
+38-5Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,47 @@
11
{
2-
"Blob-constructor.any.js": {
3-
"skip": "Depends on File API"
4-
},
52
"Blob-constructor-dom.window.js": {
63
"skip": "Depends on DOM API"
74
},
8-
"Blob-slice.any.js": {
9-
"skip": "Depends on File API"
5+
"Blob-constructor.any.js": {
6+
"fail": {
7+
"note": "Depends on File API",
8+
"expected": [
9+
"A plain object with @@iterator should be treated as a sequence for the blobParts argument.",
10+
"A plain object with @@iterator and a length property should be treated as a sequence for the blobParts argument.",
11+
"A String object should be treated as a sequence for the blobParts argument.",
12+
"A Uint8Array object should be treated as a sequence for the blobParts argument.",
13+
"Getters and value conversions should happen in order until an exception is thrown.",
14+
"Changes to the blobParts array should be reflected in the returned Blob (pop).",
15+
"Changes to the blobParts array should be reflected in the returned Blob (unshift).",
16+
"ToString should be called on elements of the blobParts array.",
17+
"ArrayBuffer elements of the blobParts array should be supported.",
18+
"Passing typed arrays as elements of the blobParts array should work.",
19+
"Passing a Float64Array as element of the blobParts array should work.",
20+
"Array with two blobs",
21+
"Array with two buffers",
22+
"Array with two bufferviews",
23+
"Array with mixed types",
24+
"options properties should be accessed in lexicographic order.",
25+
"Arguments should be evaluated from left to right.",
26+
"Passing null (index 0) for options should use the defaults.",
27+
"Passing null (index 0) for options should use the defaults (with newlines).",
28+
"Passing undefined (index 1) for options should use the defaults.",
29+
"Passing undefined (index 1) for options should use the defaults (with newlines).",
30+
"Passing object \"[object Object]\" (index 2) for options should use the defaults.",
31+
"Passing object \"[object Object]\" (index 2) for options should use the defaults (with newlines).",
32+
"Passing object \"[object Object]\" (index 3) for options should use the defaults.",
33+
"Passing object \"[object Object]\" (index 3) for options should use the defaults (with newlines).",
34+
"Passing object \"/regex/\" (index 4) for options should use the defaults.",
35+
"Passing object \"/regex/\" (index 4) for options should use the defaults (with newlines).",
36+
"Passing function \"function() {}\" (index 5) for options should use the defaults.",
37+
"Passing function \"function() {}\" (index 5) for options should use the defaults (with newlines)."
38+
]
39+
}
1040
},
1141
"Blob-in-worker.worker.js": {
1242
"skip": "Depends on Web Workers API"
43+
},
44+
"Blob-slice.any.js": {
45+
"skip": "Depends on File API"
1346
}
1447
}

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.