Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit db266df

Browse filesBrowse files
panvaMylesBorins
panva
authored and
MylesBorins
committed
test: add WPTRunner support for variants and generating WPT reports
PR-URL: #46498 Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com> Reviewed-By: Richard Lau <rlau@redhat.com>
1 parent f4fe58b commit db266df
Copy full SHA for db266df

File tree

Expand file treeCollapse file tree

4 files changed

+208
-63
lines changed
Open diff view settings
Filter options
Expand file treeCollapse file tree

4 files changed

+208
-63
lines changed
Open diff view settings
Collapse file

‎Makefile‎

Copy file name to clipboardExpand all lines: Makefile
+6Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -595,6 +595,12 @@ test-message: test-build
595595
test-wpt: all
596596
$(PYTHON) tools/test.py $(PARALLEL_ARGS) wpt
597597

598+
.PHONY: test-wpt-report
599+
test-wpt-report:
600+
$(RM) -r out/wpt
601+
mkdir -p out/wpt
602+
WPT_REPORT=1 $(PYTHON) tools/test.py --shell $(NODE) $(PARALLEL_ARGS) wpt
603+
598604
.PHONY: test-simple
599605
test-simple: | cctest # Depends on 'all'.
600606
$(PYTHON) tools/test.py $(PARALLEL_ARGS) parallel sequential
Collapse file

‎test/common/wpt.js‎

Copy file name to clipboardExpand all lines: test/common/wpt.js
+202-58Lines changed: 202 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,119 @@ const fs = require('fs');
66
const fsPromises = fs.promises;
77
const path = require('path');
88
const events = require('events');
9+
const os = require('os');
910
const { inspect } = require('util');
1011
const { Worker } = require('worker_threads');
1112

13+
function getBrowserProperties() {
14+
const { node: version } = process.versions; // e.g. 18.13.0, 20.0.0-nightly202302078e6e215481
15+
const release = /^\d+\.\d+\.\d+$/.test(version);
16+
const browser = {
17+
browser_channel: release ? 'stable' : 'experimental',
18+
browser_version: version,
19+
};
20+
21+
return browser;
22+
}
23+
24+
/**
25+
* Return one of three expected values
26+
* https://github.com/web-platform-tests/wpt/blob/1c6ff12/tools/wptrunner/wptrunner/tests/test_update.py#L953-L958
27+
*/
28+
function getOs() {
29+
switch (os.type()) {
30+
case 'Linux':
31+
return 'linux';
32+
case 'Darwin':
33+
return 'mac';
34+
case 'Windows_NT':
35+
return 'win';
36+
default:
37+
throw new Error('Unsupported os.type()');
38+
}
39+
}
40+
41+
// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L3705
42+
function sanitizeUnpairedSurrogates(str) {
43+
return str.replace(
44+
/([\ud800-\udbff]+)(?![\udc00-\udfff])|(^|[^\ud800-\udbff])([\udc00-\udfff]+)/g,
45+
function(_, low, prefix, high) {
46+
let output = prefix || ''; // Prefix may be undefined
47+
const string = low || high; // Only one of these alternates can match
48+
for (let i = 0; i < string.length; i++) {
49+
output += codeUnitStr(string[i]);
50+
}
51+
return output;
52+
});
53+
}
54+
55+
function codeUnitStr(char) {
56+
return 'U+' + char.charCodeAt(0).toString(16);
57+
}
58+
59+
class WPTReport {
60+
constructor() {
61+
this.results = [];
62+
this.time_start = Date.now();
63+
}
64+
65+
addResult(name, status) {
66+
const result = {
67+
test: name,
68+
status,
69+
subtests: [],
70+
addSubtest(name, status, message) {
71+
const subtest = {
72+
status,
73+
// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L3722
74+
name: sanitizeUnpairedSurrogates(name),
75+
};
76+
if (message) {
77+
// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L4506
78+
subtest.message = sanitizeUnpairedSurrogates(message);
79+
}
80+
this.subtests.push(subtest);
81+
return subtest;
82+
},
83+
};
84+
this.results.push(result);
85+
return result;
86+
}
87+
88+
write() {
89+
this.time_end = Date.now();
90+
this.results = this.results.filter((result) => {
91+
return result.status === 'SKIP' || result.subtests.length !== 0;
92+
}).map((result) => {
93+
const url = new URL(result.test, 'http://wpt');
94+
url.pathname = url.pathname.replace(/\.js$/, '.html');
95+
result.test = url.href.slice(url.origin.length);
96+
return result;
97+
});
98+
99+
if (fs.existsSync('out/wpt/wptreport.json')) {
100+
const prev = JSON.parse(fs.readFileSync('out/wpt/wptreport.json'));
101+
this.results = [...prev.results, ...this.results];
102+
this.time_start = prev.time_start;
103+
this.time_end = Math.max(this.time_end, prev.time_end);
104+
this.run_info = prev.run_info;
105+
} else {
106+
/**
107+
* Return required and some optional properties
108+
* https://github.com/web-platform-tests/wpt.fyi/blob/60da175/api/README.md?plain=1#L331-L335
109+
*/
110+
this.run_info = {
111+
product: 'node.js',
112+
...getBrowserProperties(),
113+
revision: process.env.WPT_REVISION || 'unknown',
114+
os: getOs(),
115+
};
116+
}
117+
118+
fs.writeFileSync('out/wpt/wptreport.json', JSON.stringify(this));
119+
}
120+
}
121+
12122
// https://github.com/web-platform-tests/wpt/blob/HEAD/resources/testharness.js
13123
// TODO: get rid of this half-baked harness in favor of the one
14124
// pulled from WPT
@@ -313,6 +423,10 @@ class WPTRunner {
313423
this.unexpectedFailures = [];
314424

315425
this.scriptsModifier = null;
426+
427+
if (process.env.WPT_REPORT != null) {
428+
this.report = new WPTReport();
429+
}
316430
}
317431

318432
/**
@@ -339,18 +453,27 @@ class WPTRunner {
339453
this.scriptsModifier = modifier;
340454
}
341455

342-
get fullInitScript() {
456+
fullInitScript(hasSubsetScript, locationSearchString) {
457+
let { initScript } = this;
458+
if (hasSubsetScript || locationSearchString) {
459+
initScript = `${initScript}\n\n//===\nglobalThis.location ||= {};`;
460+
}
461+
462+
if (locationSearchString) {
463+
initScript = `${initScript}\n\n//===\nglobalThis.location.search = "${locationSearchString}";`;
464+
}
465+
343466
if (this.globalThisInitScripts.length === null) {
344-
return this.initScript;
467+
return initScript;
345468
}
346469

347470
const globalThisInitScript = this.globalThisInitScripts.join('\n\n//===\n');
348471

349-
if (this.initScript === null) {
472+
if (initScript === null) {
350473
return globalThisInitScript;
351474
}
352475

353-
return `${globalThisInitScript}\n\n//===\n${this.initScript}`;
476+
return `${globalThisInitScript}\n\n//===\n${initScript}`;
354477
}
355478

356479
/**
@@ -455,15 +578,20 @@ class WPTRunner {
455578
for (const spec of queue) {
456579
const testFileName = spec.filename;
457580
const content = spec.getContent();
458-
const meta = spec.title = this.getMeta(content);
581+
const meta = spec.meta = this.getMeta(content);
459582

460583
const absolutePath = spec.getAbsolutePath();
461584
const relativePath = spec.getRelativePath();
462585
const harnessPath = fixtures.path('wpt', 'resources', 'testharness.js');
463586
const scriptsToRun = [];
587+
let hasSubsetScript = false;
588+
464589
// Scripts specified with the `// META: script=` header
465590
if (meta.script) {
466591
for (const script of meta.script) {
592+
if (script === '/common/subset-tests.js' || script === '/common/subset-tests-by-key.js') {
593+
hasSubsetScript = true;
594+
}
467595
const obj = {
468596
filename: this.resource.toRealFilePath(relativePath, script),
469597
code: this.resource.read(relativePath, script, false),
@@ -480,54 +608,65 @@ class WPTRunner {
480608
this.scriptsModifier?.(obj);
481609
scriptsToRun.push(obj);
482610

483-
const workerPath = path.join(__dirname, 'wpt/worker.js');
484-
const worker = new Worker(workerPath, {
485-
execArgv: this.flags,
486-
workerData: {
487-
testRelativePath: relativePath,
488-
wptRunner: __filename,
489-
wptPath: this.path,
490-
initScript: this.fullInitScript,
491-
harness: {
492-
code: fs.readFileSync(harnessPath, 'utf8'),
493-
filename: harnessPath,
611+
/**
612+
* Example test with no META variant
613+
* https://github.com/nodejs/node/blob/03854f6/test/fixtures/wpt/WebCryptoAPI/sign_verify/hmac.https.any.js#L1-L4
614+
*
615+
* Example test with multiple META variants
616+
* https://github.com/nodejs/node/blob/03854f6/test/fixtures/wpt/WebCryptoAPI/generateKey/successes_RSASSA-PKCS1-v1_5.https.any.js#L1-L9
617+
*/
618+
for (const variant of meta.variant || ['']) {
619+
const workerPath = path.join(__dirname, 'wpt/worker.js');
620+
const worker = new Worker(workerPath, {
621+
execArgv: this.flags,
622+
workerData: {
623+
testRelativePath: relativePath,
624+
wptRunner: __filename,
625+
wptPath: this.path,
626+
initScript: this.fullInitScript(hasSubsetScript, variant),
627+
harness: {
628+
code: fs.readFileSync(harnessPath, 'utf8'),
629+
filename: harnessPath,
630+
},
631+
scriptsToRun,
494632
},
495-
scriptsToRun,
496-
},
497-
});
498-
this.workers.set(testFileName, worker);
499-
500-
worker.on('message', (message) => {
501-
switch (message.type) {
502-
case 'result':
503-
return this.resultCallback(testFileName, message.result);
504-
case 'completion':
505-
return this.completionCallback(testFileName, message.status);
506-
default:
507-
throw new Error(`Unexpected message from worker: ${message.type}`);
508-
}
509-
});
633+
});
634+
this.workers.set(testFileName, worker);
635+
636+
let reportResult;
637+
worker.on('message', (message) => {
638+
switch (message.type) {
639+
case 'result':
640+
reportResult ||= this.report?.addResult(`/${relativePath}${variant}`, 'OK');
641+
return this.resultCallback(testFileName, message.result, reportResult);
642+
case 'completion':
643+
return this.completionCallback(testFileName, message.status);
644+
default:
645+
throw new Error(`Unexpected message from worker: ${message.type}`);
646+
}
647+
});
510648

511-
worker.on('error', (err) => {
512-
if (!this.inProgress.has(testFileName)) {
513-
// The test is already finished. Ignore errors that occur after it.
514-
// This can happen normally, for example in timers tests.
515-
return;
516-
}
517-
this.fail(
518-
testFileName,
519-
{
520-
status: NODE_UNCAUGHT,
521-
name: 'evaluation in WPTRunner.runJsTests()',
522-
message: err.message,
523-
stack: inspect(err),
524-
},
525-
kUncaught,
526-
);
527-
this.inProgress.delete(testFileName);
528-
});
649+
worker.on('error', (err) => {
650+
if (!this.inProgress.has(testFileName)) {
651+
// The test is already finished. Ignore errors that occur after it.
652+
// This can happen normally, for example in timers tests.
653+
return;
654+
}
655+
this.fail(
656+
testFileName,
657+
{
658+
status: NODE_UNCAUGHT,
659+
name: 'evaluation in WPTRunner.runJsTests()',
660+
message: err.message,
661+
stack: inspect(err),
662+
},
663+
kUncaught,
664+
);
665+
this.inProgress.delete(testFileName);
666+
});
529667

530-
await events.once(worker, 'exit').catch(() => {});
668+
await events.once(worker, 'exit').catch(() => {});
669+
}
531670
}
532671

533672
process.on('exit', () => {
@@ -587,6 +726,8 @@ class WPTRunner {
587726
}
588727
}
589728

729+
this.report?.write();
730+
590731
const ran = queue.length;
591732
const total = ran + skipped;
592733
const passed = ran - expectedFailures - failures.length;
@@ -611,8 +752,7 @@ class WPTRunner {
611752

612753
getTestTitle(filename) {
613754
const spec = this.specMap.get(filename);
614-
const title = spec.meta && spec.meta.title;
615-
return title ? `${filename} : ${title}` : filename;
755+
return spec.meta?.title || filename;
616756
}
617757

618758
// Map WPT test status to strings
@@ -638,14 +778,14 @@ class WPTRunner {
638778
* @param {string} filename
639779
* @param {Test} test The Test object returned by WPT harness
640780
*/
641-
resultCallback(filename, test) {
781+
resultCallback(filename, test, reportResult) {
642782
const status = this.getTestStatus(test.status);
643783
const title = this.getTestTitle(filename);
644784
console.log(`---- ${title} ----`);
645785
if (status !== kPass) {
646-
this.fail(filename, test, status);
786+
this.fail(filename, test, status, reportResult);
647787
} else {
648-
this.succeed(filename, test, status);
788+
this.succeed(filename, test, status, reportResult);
649789
}
650790
}
651791

@@ -693,11 +833,12 @@ class WPTRunner {
693833
}
694834
}
695835

696-
succeed(filename, test, status) {
836+
succeed(filename, test, status, reportResult) {
697837
console.log(`[${status.toUpperCase()}] ${test.name}`);
838+
reportResult?.addSubtest(test.name, 'PASS');
698839
}
699840

700-
fail(filename, test, status) {
841+
fail(filename, test, status, reportResult) {
701842
const spec = this.specMap.get(filename);
702843
const expected = spec.failedTests.includes(test.name);
703844
if (expected) {
@@ -713,6 +854,9 @@ class WPTRunner {
713854
const command = `${process.execPath} ${process.execArgv}` +
714855
` ${require.main.filename} ${filename}`;
715856
console.log(`Command: ${command}\n`);
857+
858+
reportResult?.addSubtest(test.name, 'FAIL', test.message);
859+
716860
this.addTestResult(filename, {
717861
name: test.name,
718862
expected,
@@ -742,7 +886,7 @@ class WPTRunner {
742886
const parts = match.match(/\/\/ META: ([^=]+?)=(.+)/);
743887
const key = parts[1];
744888
const value = parts[2];
745-
if (key === 'script') {
889+
if (key === 'script' || key === 'variant') {
746890
if (result[key]) {
747891
result[key].push(value);
748892
} else {
Collapse file

‎test/wpt/test-encoding.js‎

Copy file name to clipboardExpand all lines: test/wpt/test-encoding.js
-1Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ const { WPTRunner } = require('../common/wpt');
44
const runner = new WPTRunner('encoding');
55

66
runner.setInitScript(`
7-
globalThis.location ||= {};
87
const { MessageChannel } = require('worker_threads');
98
global.MessageChannel = MessageChannel;
109
`);
Collapse file

‎test/wpt/test-webcrypto.js‎

Copy file name to clipboardExpand all lines: test/wpt/test-webcrypto.js
-4Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,6 @@ const { WPTRunner } = require('../common/wpt');
88

99
const runner = new WPTRunner('WebCryptoAPI');
1010

11-
runner.setInitScript(`
12-
global.location = {};
13-
`);
14-
1511
runner.pretendGlobalThisAs('Window');
1612

1713
runner.runJsTests();

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.