Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 3c8f700

Browse filesBrowse files
addaleaxaduh95
addaleax
authored and
aduh95
committed
src: track allocations made by zstd streams
This is both valuable as a diagnostic tool and as a way to inform the JS runtime about external allocations. Currently, this is a feature only enabled in the statically linked builds of zstd, so with `--shared-zstd`, we fall back to the non-tracking variant. PR-URL: #61717 Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com> Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com> Reviewed-By: Rafael Gonzaga <rafael.nunu@hotmail.com>
1 parent 94dbb36 commit 3c8f700
Copy full SHA for 3c8f700

3 files changed

+72Lines changed: 72 additions & 0 deletions

File tree

Expand file treeCollapse file tree
Open diff view settings
Filter options
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎node.gypi‎

Copy file name to clipboardExpand all lines: node.gypi
+1Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,7 @@
248248

249249
[ 'node_shared_zstd=="false"', {
250250
'dependencies': [ 'deps/zstd/zstd.gyp:zstd' ],
251+
'defines': [ 'NODE_BUNDLED_ZSTD' ],
251252
}],
252253

253254
[ 'OS=="mac"', {
Collapse file

‎src/node_zlib.cc‎

Copy file name to clipboardExpand all lines: src/node_zlib.cc
+22Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,10 @@
1919
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
2020
// USE OR OTHER DEALINGS IN THE SOFTWARE.
2121

22+
#ifdef NODE_BUNDLED_ZSTD
23+
#define ZSTD_STATIC_LINKING_ONLY
24+
#endif
25+
2226
#include "memory_tracker-inl.h"
2327
#include "node.h"
2428
#include "node_buffer.h"
@@ -1549,7 +1553,16 @@ void ZstdCompressContext::Close() {
15491553
CompressionError ZstdCompressContext::Init(uint64_t pledged_src_size,
15501554
std::string_view dictionary) {
15511555
pledged_src_size_ = pledged_src_size;
1556+
#ifdef NODE_BUNDLED_ZSTD
1557+
ZSTD_customMem custom_mem = {
1558+
CompressionStreamMemoryOwner::AllocForBrotli,
1559+
CompressionStreamMemoryOwner::FreeForZlib,
1560+
CompressionStream<ZstdCompressContext>::AllocatorOpaquePointerForContext(
1561+
this)};
1562+
cctx_.reset(ZSTD_createCCtx_advanced(custom_mem));
1563+
#else
15521564
cctx_.reset(ZSTD_createCCtx());
1565+
#endif
15531566
if (!cctx_) {
15541567
return CompressionError("Could not initialize zstd instance",
15551568
"ERR_ZLIB_INITIALIZATION_FAILED",
@@ -1604,7 +1617,16 @@ void ZstdDecompressContext::Close() {
16041617

16051618
CompressionError ZstdDecompressContext::Init(uint64_t pledged_src_size,
16061619
std::string_view dictionary) {
1620+
#ifdef NODE_BUNDLED_ZSTD
1621+
ZSTD_customMem custom_mem = {
1622+
CompressionStreamMemoryOwner::AllocForBrotli,
1623+
CompressionStreamMemoryOwner::FreeForZlib,
1624+
CompressionStream<
1625+
ZstdDecompressContext>::AllocatorOpaquePointerForContext(this)};
1626+
dctx_.reset(ZSTD_createDCtx_advanced(custom_mem));
1627+
#else
16071628
dctx_.reset(ZSTD_createDCtx());
1629+
#endif
16081630
if (!dctx_) {
16091631
return CompressionError("Could not initialize zstd instance",
16101632
"ERR_ZLIB_INITIALIZATION_FAILED",
Collapse file

‎test/pummel/test-heapdump-zstd.js‎

Copy file name to clipboard
+49Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
'use strict';
2+
// This tests heap snapshot integration of zlib stream.
3+
4+
const common = require('../common');
5+
const assert = require('assert');
6+
const { validateByRetainingPath, validateByRetainingPathFromNodes } = require('../common/heap');
7+
const zlib = require('zlib');
8+
9+
// Before zstd stream is created, no ZstdStream should be created.
10+
{
11+
const nodes = validateByRetainingPath('Node / ZstdStream', []);
12+
assert.strictEqual(nodes.length, 0);
13+
}
14+
15+
const compress = zlib.createZstdCompress();
16+
17+
// After zstd stream is created, a ZstdStream should be created.
18+
{
19+
const streams = validateByRetainingPath('Node / ZstdStream', []);
20+
validateByRetainingPathFromNodes(streams, 'Node / ZstdStream', [
21+
{ node_name: 'ZstdCompress', edge_name: 'native_to_javascript' },
22+
]);
23+
const withMemory = validateByRetainingPathFromNodes(streams, 'Node / ZstdStream', [
24+
{ node_name: 'Node / zlib_memory', edge_name: 'zlib_memory' },
25+
], true);
26+
if (process.config.variables.node_shared_zstd === true) {
27+
assert.strictEqual(withMemory.length, 0);
28+
} else {
29+
assert.strictEqual(withMemory.length, 1);
30+
// Between 1KB and 1MB (measured value was around ~5kB)
31+
assert.ok(withMemory[0].self_size > 1024);
32+
assert.ok(withMemory[0].self_size < 1024 * 1024);
33+
}
34+
}
35+
36+
// After zstd stream is written, zlib_memory is significantly larger.
37+
compress.write('hello world', common.mustCall(() => {
38+
const streams = validateByRetainingPath('Node / ZstdStream', []);
39+
const withMemory = validateByRetainingPathFromNodes(streams, 'Node / ZstdStream', [
40+
{ node_name: 'Node / zlib_memory', edge_name: 'zlib_memory' },
41+
], true);
42+
if (process.config.variables.node_shared_zstd === true) {
43+
assert.strictEqual(withMemory.length, 0);
44+
} else {
45+
assert.strictEqual(withMemory.length, 1);
46+
// More than 1MB
47+
assert.ok(withMemory[0].self_size > 1024 * 1024);
48+
}
49+
}));

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.