Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 5d5c848

Browse filesBrowse files
aduh95aduh95
committed
tools: run CI with shared libs on GHA
PR-URL: #60121 Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com> Reviewed-By: Chengzhong Wu <legendecas@gmail.com>
1 parent 7a6542c commit 5d5c848
Copy full SHA for 5d5c848

File tree

Expand file treeCollapse file tree

10 files changed

+359
-1
lines changed
Open diff view settings
Filter options
Expand file treeCollapse file tree

10 files changed

+359
-1
lines changed
Open diff view settings
Collapse file

‎.github/workflows/linters.yml‎

Copy file name to clipboardExpand all lines: .github/workflows/linters.yml
+16Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,22 @@ jobs:
140140
NODE=$(command -v node) make lint-md
141141
env:
142142
NODE_RELEASED_VERSIONS: ${{ steps.get-released-versions.outputs.NODE_RELEASED_VERSIONS }}
143+
lint-nix:
144+
if: github.event.pull_request.draft == false
145+
runs-on: ubuntu-latest
146+
steps:
147+
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
148+
with:
149+
persist-credentials: false
150+
sparse-checkout: '*.nix'
151+
sparse-checkout-cone-mode: false
152+
- uses: cachix/install-nix-action@7be5dee1421f63d07e71ce6e0a9f8a4b07c2a487 # v31.6.1
153+
- name: Lint Nix files
154+
run: nix-shell -I nixpkgs=./tools/nix/pkgs.nix -p 'nixfmt-tree' --run 'treefmt --quiet --fail-on-change'
155+
- if: ${{ failure() }}
156+
name: Show diff
157+
run: git --no-pager diff
158+
143159
lint-py:
144160
if: github.event.pull_request.draft == false
145161
runs-on: ubuntu-latest
Collapse file

‎.github/workflows/test-linux.yml‎

Copy file name to clipboardExpand all lines: .github/workflows/test-linux.yml
+2Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ on:
55
paths-ignore:
66
- .mailmap
77
- README.md
8+
- '*.nix'
89
- .github/**
910
- '!.github/workflows/test-linux.yml'
1011
types: [opened, synchronize, reopened, ready_for_review]
@@ -17,6 +18,7 @@ on:
1718
paths-ignore:
1819
- .mailmap
1920
- README.md
21+
- '*.nix'
2022
- .github/**
2123
- '!.github/workflows/test-linux.yml'
2224

Collapse file

‎.github/workflows/test-macos.yml‎

Copy file name to clipboardExpand all lines: .github/workflows/test-macos.yml
+2Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ on:
66
paths-ignore:
77
- .mailmap
88
- '**.md'
9+
- '*.nix'
910
- AUTHORS
1011
- doc/**
1112
- .github/**
@@ -19,6 +20,7 @@ on:
1920
paths-ignore:
2021
- .mailmap
2122
- '**.md'
23+
- '*.nix'
2224
- AUTHORS
2325
- doc/**
2426
- .github/**
Collapse file

‎.github/workflows/test-shared.yml‎

Copy file name to clipboard
+83Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
name: Test Shared librairies
2+
3+
on:
4+
pull_request:
5+
paths-ignore:
6+
- .mailmap
7+
- '**.md'
8+
- AUTHORS
9+
- doc/**
10+
- .github/**
11+
- '!.github/workflows/test-shared.yml'
12+
types: [opened, synchronize, reopened, ready_for_review]
13+
push:
14+
branches:
15+
- main
16+
- canary
17+
- v[0-9]+.x-staging
18+
- v[0-9]+.x
19+
paths-ignore:
20+
- .mailmap
21+
- '**.md'
22+
- AUTHORS
23+
- doc/**
24+
- .github/**
25+
- '!.github/workflows/test-shared.yml'
26+
27+
concurrency:
28+
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
29+
cancel-in-progress: true
30+
31+
env:
32+
FLAKY_TESTS: keep_retrying
33+
34+
permissions:
35+
contents: read
36+
37+
jobs:
38+
build:
39+
strategy:
40+
fail-fast: false
41+
matrix:
42+
include:
43+
- runner: ubuntu-24.04
44+
system: x86_64-linux
45+
- runner: ubuntu-24.04-arm
46+
system: aarch64-linux
47+
- runner: macos-13
48+
system: x86_64-darwin
49+
- runner: macos-latest
50+
system: aarch64-darwin
51+
name: '${{ matrix.system }}: with shared libraries'
52+
runs-on: ${{ matrix.runner }}
53+
steps:
54+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
55+
with:
56+
persist-credentials: false
57+
58+
- uses: cachix/install-nix-action@7be5dee1421f63d07e71ce6e0a9f8a4b07c2a487 # v31.6.1
59+
with:
60+
extra_nix_config: sandbox = true
61+
62+
- name: Configure sccache
63+
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
64+
with:
65+
script: |
66+
core.exportVariable('SCCACHE_GHA_VERSION', 'on');
67+
core.exportVariable('ACTIONS_CACHE_SERVICE_V2', 'on');
68+
core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
69+
core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
70+
71+
- name: Build Node.js and run tests
72+
run: |
73+
nix-shell \
74+
-I nixpkgs=./tools/nix/pkgs.nix \
75+
--pure --keep FLAKY_TESTS \
76+
--keep SCCACHE_GHA_VERSION --keep ACTIONS_CACHE_SERVICE_V2 --keep ACTIONS_RESULTS_URL --keep ACTIONS_RUNTIME_TOKEN \
77+
--arg loadJSBuiltinsDynamically false \
78+
--arg ccache '(import <nixpkgs> {}).sccache' \
79+
--arg devTools '[]' \
80+
--arg benchmarkTools '[]' \
81+
--run '
82+
make run-ci -j4 V=1 TEST_CI_ARGS="-p actions --measure-flakiness 9 --skip-tests=$CI_SKIP_TESTS"
83+
'
Collapse file

‎.github/workflows/tools.yml‎

Copy file name to clipboardExpand all lines: .github/workflows/tools.yml
+12Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ on:
3030
- llhttp
3131
- minimatch
3232
- nbytes
33+
- nixpkgs-unstable
3334
- nghttp2
3435
- nghttp3
3536
- ngtcp2
@@ -190,6 +191,14 @@ jobs:
190191
cat temp-output
191192
tail -n1 temp-output | grep "NEW_VERSION=" >> "$GITHUB_ENV" || true
192193
rm temp-output
194+
- id: nixpkgs-unstable
195+
subsystem: tools
196+
label: tools
197+
run: |
198+
./tools/dep_updaters/update-nixpkgs-pin.sh > temp-output
199+
cat temp-output
200+
tail -n1 temp-output | grep "NEW_VERSION=" >> "$GITHUB_ENV" || true
201+
rm temp-output
193202
- id: nghttp2
194203
subsystem: deps
195204
label: dependencies
@@ -292,6 +301,9 @@ jobs:
292301
with:
293302
python-version: ${{ env.PYTHON_VERSION }}
294303
allow-prereleases: true
304+
- name: Set up Nix
305+
if: matrix.id == 'nixpkgs-unstable' && (github.event_name == 'schedule' || inputs.id == 'all' || inputs.id == matrix.id)
306+
uses: cachix/install-nix-action@7be5dee1421f63d07e71ce6e0a9f8a4b07c2a487 # v31.6.1
295307
- run: ${{ matrix.run }}
296308
if: github.event_name == 'schedule' || inputs.id == 'all' || inputs.id == matrix.id
297309
env:
Collapse file

‎BUILDING.md‎

Copy file name to clipboardExpand all lines: BUILDING.md
+70-1Lines changed: 70 additions & 1 deletion
  • Display the source diff
  • Display the rich diff
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,7 @@ Consult previous versions of this document for older versions of Node.js:
239239

240240
Installation via Linux package manager can be achieved with:
241241

242+
* Nix, NixOS: `nix-shell`
242243
* Ubuntu, Debian: `sudo apt-get install python3 g++-12 gcc-12 make python3-pip`
243244
* Fedora: `sudo dnf install python3 gcc-c++ make python3-pip`
244245
* CentOS and RHEL: `sudo yum install python3 gcc-c++ make python3-pip`
@@ -259,6 +260,75 @@ installed, you can find them under the menu `Xcode -> Open Developer Tool ->
259260
More Developer Tools...`. This step will install `clang`, `clang++`, and
260261
`make`.
261262

263+
#### Nix integration
264+
265+
If you are using Nix and direnv, you can use the following to get started:
266+
267+
```bash
268+
echo 'use_nix --arg sharedLibDeps {} --argstr icu small' > .envrc
269+
direnv allow .
270+
make build-ci -j12
271+
```
272+
273+
The use of `make build-ci` is to ensure you are using the `CONFIG_FLAGS`
274+
environment variable. You can also specify it manually:
275+
276+
```bash
277+
./configure $CONFIG_FLAGS
278+
make -j12
279+
```
280+
281+
Passing the `--arg sharedLibDeps {}` instructs direnv and Nix to generate an
282+
environment that uses the vendored-in native dependencies. Using the vendored-in
283+
dependencies result in a result closer to the official binaries, the tradeoff
284+
being the build will take longer to complete as you'd have to build those
285+
dependencies instead of using the cached ones from the Nix cache. You can omit
286+
that flag to use all the shared dependencies, or specify only some dependencies:
287+
288+
```bash
289+
cat -> .envrc <<'EOF'
290+
use nix --arg sharedLibDeps '{
291+
inherit (import <nixpkgs> {})
292+
openssl
293+
zlib
294+
;
295+
}'
296+
EOF
297+
```
298+
299+
Passing the `--argstr icu small` instructs direnv and Nix to pass `--with-intl=small` in
300+
the `CONFIG_FLAGS` environment variable. If you omit this, the prebuilt ICU from Nix cache
301+
will be used, which should speed up greatly compilation time.
302+
303+
The use of `direnv` is completely optional, you can also use `nix-shell` directly,
304+
e.g. here's a command you can use to build a binary for benchmarking purposes:
305+
306+
```bash
307+
# Passing `--arg loadJSBuiltinsDynamically false` to instruct the compiler to
308+
# embed the JS core files so it is no longer affected by local changes
309+
# (necessary for getting useful benchmark results).
310+
# Passing `--arg devTools '[]' --arg benchmarkTools '[]'` since we don't need
311+
# those to build node.
312+
nix-shell \
313+
--arg loadJSBuiltinsDynamically false \
314+
--arg devTools '[]' --arg benchmarkTools '[]' \
315+
--run 'make build-ci -j12'
316+
317+
mv out/Release/node ./node_old
318+
319+
# ...
320+
# Make your local changes, and re-build node
321+
322+
nix-shell \
323+
--arg loadJSBuiltinsDynamically false \
324+
--arg devTools '[]' --arg benchmarkTools '[]' \
325+
--run 'make build-ci -j12'
326+
327+
nix-shell --pure --run './node benchmark/compare.js --old ./node_old --new ./node http | Rscript benchmark/compare.R'
328+
```
329+
330+
There are additional attributes you can pass, see `shell.nix` file for more details.
331+
262332
#### Building Node.js
263333

264334
If the path to your build directory contains a space, the build will likely
@@ -267,7 +337,6 @@ fail.
267337
To build Node.js:
268338

269339
```bash
270-
export CXX=g++-12
271340
./configure
272341
make -j4
273342
```
Collapse file

‎doc/contributing/writing-and-running-benchmarks.md‎

Copy file name to clipboardExpand all lines: doc/contributing/writing-and-running-benchmarks.md
+4Lines changed: 4 additions & 0 deletions
  • Display the source diff
  • Display the rich diff
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ Basic Unix tools are required for some benchmarks.
2626
[Git for Windows][git-for-windows] includes Git Bash and the necessary tools,
2727
which need to be included in the global Windows `PATH`.
2828

29+
If you are using Nix, all the required tools are already listed in the
30+
`benchmarkTools` argument of the `shell.nix` file, so you can skip those
31+
prerequesites.
32+
2933
### HTTP benchmark requirements
3034

3135
Most of the HTTP benchmarks require a benchmarker to be installed. This can be
Collapse file

‎shell.nix‎

Copy file name to clipboard
+120Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
{
2+
pkgs ? import "${./tools/nix/pkgs.nix}" { },
3+
loadJSBuiltinsDynamically ? true, # Load `lib/**.js` from disk instead of embedding
4+
ncu-path ? null, # Provide this if you want to use a local version of NCU
5+
icu ? pkgs.icu,
6+
sharedLibDeps ? {
7+
inherit (pkgs)
8+
ada
9+
brotli
10+
c-ares
11+
libuv
12+
nghttp2
13+
nghttp3
14+
ngtcp2
15+
openssl
16+
simdjson
17+
simdutf
18+
sqlite
19+
uvwasi
20+
zlib
21+
zstd
22+
;
23+
http-parser = pkgs.llhttp;
24+
},
25+
ccache ? pkgs.ccache,
26+
ninja ? pkgs.ninja,
27+
devTools ? [
28+
pkgs.curl
29+
pkgs.gh
30+
pkgs.git
31+
pkgs.jq
32+
pkgs.shellcheck
33+
]
34+
++ (
35+
if (ncu-path == null) then
36+
[ pkgs.node-core-utils ]
37+
else
38+
[
39+
(pkgs.writeShellScriptBin "git-node" "exec \"${ncu-path}/bin/git-node.js\" \"$@\"")
40+
(pkgs.writeShellScriptBin "ncu-ci" "exec \"${ncu-path}/bin/ncu-ci.js\" \"$@\"")
41+
(pkgs.writeShellScriptBin "ncu-config" "exec \"${ncu-path}/bin/ncu-config.js\" \"$@\"")
42+
]
43+
),
44+
benchmarkTools ? [
45+
pkgs.R
46+
pkgs.rPackages.ggplot2
47+
pkgs.rPackages.plyr
48+
pkgs.wrk
49+
],
50+
extraConfigFlags ? [
51+
"--without-npm"
52+
"--debug-node"
53+
],
54+
}:
55+
56+
let
57+
useSharedICU = if builtins.isString icu then icu == "system" else icu != null;
58+
useSharedAda = builtins.hasAttr "ada" sharedLibDeps;
59+
useSharedOpenSSL = builtins.hasAttr "openssl" sharedLibDeps;
60+
in
61+
pkgs.mkShell {
62+
inherit (pkgs.nodejs_latest) nativeBuildInputs;
63+
64+
buildInputs = builtins.attrValues sharedLibDeps ++ pkgs.lib.optionals useSharedICU [ icu ];
65+
66+
packages = [
67+
ccache
68+
]
69+
++ devTools
70+
++ benchmarkTools;
71+
72+
shellHook =
73+
if (ccache != null) then
74+
''
75+
export CC="${pkgs.lib.getExe ccache} $CC"
76+
export CXX="${pkgs.lib.getExe ccache} $CXX"
77+
''
78+
else
79+
"";
80+
81+
BUILD_WITH = if (ninja != null) then "ninja" else "make";
82+
NINJA = if (ninja != null) then "${pkgs.lib.getExe ninja}" else "";
83+
CI_SKIP_TESTS = pkgs.lib.concatStringsSep "," (
84+
[ ]
85+
++ pkgs.lib.optionals useSharedAda [
86+
# Different versions of Ada affect the WPT tests
87+
"test-url"
88+
]
89+
++ pkgs.lib.optionals useSharedOpenSSL [
90+
# Path to the openssl.cnf is different from the expected one
91+
"test-strace-openat-openssl"
92+
]
93+
);
94+
CONFIG_FLAGS = builtins.toString (
95+
[
96+
(
97+
if icu == null then
98+
"--without-intl"
99+
else
100+
"--with-intl=${if useSharedICU then "system" else icu}-icu"
101+
)
102+
]
103+
++ extraConfigFlags
104+
++ pkgs.lib.optionals (ninja != null) [
105+
"--ninja"
106+
]
107+
++ pkgs.lib.optionals loadJSBuiltinsDynamically [
108+
"--node-builtin-modules-path=${builtins.toString ./.}"
109+
]
110+
++ pkgs.lib.concatMap (name: [
111+
"--shared-${builtins.replaceStrings [ "c-ares" ] [ "cares" ] name}"
112+
"--shared-${builtins.replaceStrings [ "c-ares" ] [ "cares" ] name}-libpath=${
113+
pkgs.lib.getLib sharedLibDeps.${name}
114+
}/lib"
115+
"--shared-${builtins.replaceStrings [ "c-ares" ] [ "cares" ] name}-include=${
116+
pkgs.lib.getInclude sharedLibDeps.${name}
117+
}/include"
118+
]) (builtins.attrNames sharedLibDeps)
119+
);
120+
}

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.