Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Appearance settings

Commit 896017b

Browse filesBrowse files
addaleaxtargos
authored andcommitted
build: build addon tests in parallel
Use a JS script to build addons rather than a shell command embedded in the Makefile, because parallelizing is hard in sh and easy in JS. PR-URL: #21155 Reviewed-By: Ben Noordhuis <info@bnoordhuis.nl> Reviewed-By: Colin Ihrig <cjihrig@gmail.com> Reviewed-By: Joyee Cheung <joyeec9h3@gmail.com> Reviewed-By: James M Snell <jasnell@gmail.com>
1 parent a5c386d commit 896017b
Copy full SHA for 896017b

File tree

Expand file treeCollapse file tree

2 files changed

+67
-30
lines changed
Open diff view settings
Filter options
Expand file treeCollapse file tree

2 files changed

+67
-30
lines changed
Open diff view settings
Collapse file

‎Makefile‎

Copy file name to clipboardExpand all lines: Makefile
+9-30Lines changed: 9 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -315,25 +315,14 @@ ADDONS_BINDING_SOURCES := \
315315
# Depends on node-gyp package.json so that build-addons is (re)executed when
316316
# node-gyp is updated as part of an npm update.
317317
test/addons/.buildstamp: config.gypi \
318-
deps/npm/node_modules/node-gyp/package.json \
318+
deps/npm/node_modules/node-gyp/package.json tools/build-addons.js \
319319
$(ADDONS_BINDING_GYPS) $(ADDONS_BINDING_SOURCES) \
320320
deps/uv/include/*.h deps/v8/include/*.h \
321321
src/node.h src/node_buffer.h src/node_object_wrap.h src/node_version.h \
322322
test/addons/.docbuildstamp
323-
# Cannot use $(wildcard test/addons/*/) here, it's evaluated before
324-
# embedded addons have been generated from the documentation.
325-
# Ignore folders without binding.gyp
326-
# (https://github.com/nodejs/node/issues/14843)
327-
@for dirname in test/addons/*/; do \
328-
if [ ! -f "$$PWD/$${dirname}binding.gyp" ]; then \
329-
continue; fi ; \
330-
printf "\nBuilding addon $$PWD/$$dirname\n" ; \
331-
env MAKEFLAGS="-j1" $(NODE) deps/npm/node_modules/node-gyp/bin/node-gyp \
332-
--loglevel=$(LOGLEVEL) rebuild \
333-
--python="$(PYTHON)" \
334-
--directory="$$PWD/$$dirname" \
335-
--nodedir="$$PWD" || exit 1 ; \
336-
done
323+
env npm_config_loglevel=$(LOGLEVEL) npm_config_nodedir="$$PWD" \
324+
npm_config_python="$(PYTHON)" $(NODE) "$$PWD/tools/build-addons" \
325+
"$$PWD/deps/npm/node_modules/node-gyp/bin/node-gyp.js" "$$PWD/test/addons"
337326
touch $@
338327

339328
.PHONY: build-addons
@@ -355,25 +344,15 @@ ADDONS_NAPI_BINDING_SOURCES := \
355344

356345
# Implicitly depends on $(NODE_EXE), see the build-addons-napi rule for rationale.
357346
test/addons-napi/.buildstamp: config.gypi \
358-
deps/npm/node_modules/node-gyp/package.json \
347+
deps/npm/node_modules/node-gyp/package.json tools/build-addons.js \
359348
$(ADDONS_NAPI_BINDING_GYPS) $(ADDONS_NAPI_BINDING_SOURCES) \
360349
deps/uv/include/*.h deps/v8/include/*.h \
361350
src/node.h src/node_buffer.h src/node_object_wrap.h src/node_version.h \
362351
src/node_api.h src/node_api_types.h
363-
# Cannot use $(wildcard test/addons-napi/*/) here, it's evaluated before
364-
# embedded addons have been generated from the documentation.
365-
# Ignore folders without binding.gyp
366-
# (https://github.com/nodejs/node/issues/14843)
367-
@for dirname in test/addons-napi/*/; do \
368-
if [ ! -f "$$PWD/$${dirname}binding.gyp" ]; then \
369-
continue; fi ; \
370-
printf "\nBuilding addon $$PWD/$$dirname\n" ; \
371-
env MAKEFLAGS="-j1" $(NODE) deps/npm/node_modules/node-gyp/bin/node-gyp \
372-
--loglevel=$(LOGLEVEL) rebuild \
373-
--python="$(PYTHON)" \
374-
--directory="$$PWD/$$dirname" \
375-
--nodedir="$$PWD" || exit 1 ; \
376-
done
352+
env npm_config_loglevel=$(LOGLEVEL) npm_config_nodedir="$$PWD" \
353+
npm_config_python="$(PYTHON)" $(NODE) "$$PWD/tools/build-addons" \
354+
"$$PWD/deps/npm/node_modules/node-gyp/bin/node-gyp.js" \
355+
"$$PWD/test/addons-napi"
377356
touch $@
378357

379358
.PHONY: build-addons-napi
Collapse file

‎tools/build-addons.js‎

Copy file name to clipboard
+58Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
'use strict';
2+
3+
// Usage: e.g. node build-addons.js <path to node-gyp> <directory>
4+
5+
const child_process = require('child_process');
6+
const path = require('path');
7+
const fs = require('fs').promises;
8+
const util = require('util');
9+
10+
const execFile = util.promisify(child_process.execFile);
11+
12+
const parallelization = +process.env.JOBS || require('os').cpus().length;
13+
const nodeGyp = process.argv[2];
14+
15+
async function runner(directoryQueue) {
16+
if (directoryQueue.length === 0)
17+
return;
18+
19+
const dir = directoryQueue.shift();
20+
const next = () => runner(directoryQueue);
21+
22+
try {
23+
// Only run for directories that have a `binding.gyp`.
24+
// (https://github.com/nodejs/node/issues/14843)
25+
await fs.stat(path.join(dir, 'binding.gyp'));
26+
} catch (err) {
27+
if (err.code === 'ENOENT' || err.code === 'ENOTDIR')
28+
return next();
29+
throw err;
30+
}
31+
32+
console.log(`Building addon in ${dir}`);
33+
const { stdout, stderr } =
34+
await execFile(process.execPath, [nodeGyp, 'rebuild', `--directory=${dir}`],
35+
{
36+
stdio: 'inherit',
37+
env: { ...process.env, MAKEFLAGS: '-j1' }
38+
});
39+
40+
// We buffer the output and print it out once the process is done in order
41+
// to avoid interleaved output from multiple builds running at once.
42+
process.stdout.write(stdout);
43+
process.stderr.write(stderr);
44+
45+
return next();
46+
}
47+
48+
async function main(directory) {
49+
const directoryQueue = (await fs.readdir(directory))
50+
.map((subdir) => path.join(directory, subdir));
51+
52+
const runners = [];
53+
for (let i = 0; i < parallelization; ++i)
54+
runners.push(runner(directoryQueue));
55+
return Promise.all(runners);
56+
}
57+
58+
main(process.argv[3]).catch((err) => setImmediate(() => { throw err; }));

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.