Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit f2ce2ca

Browse filesBrowse files
alexander-akaitalexander-akait
committed
refactor!: use xxhash64 by default for [hash]/[contenthash] and getHashDigest API
1 parent 10503d6 commit f2ce2ca
Copy full SHA for f2ce2ca

File tree

Expand file treeCollapse file tree

4 files changed

+222
-43
lines changed
Filter options
Expand file treeCollapse file tree

4 files changed

+222
-43
lines changed

‎README.md

Copy file name to clipboardExpand all lines: README.md
+6-6Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -77,14 +77,14 @@ The following tokens are replaced in the `name` parameter:
7777
- `[path]` the path of the resource relative to the `context` query parameter or option.
7878
- `[folder]` the folder the resource is in
7979
- `[query]` the queryof the resource, i.e. `?foo=bar`
80-
- `[contenthash]` the hash of `options.content` (Buffer) (by default it's the hex digest of the md4 hash)
80+
- `[contenthash]` the hash of `options.content` (Buffer) (by default it's the hex digest of the `xxhash64` hash)
8181
- `[<hashType>:contenthash:<digestType>:<length>]` optionally one can configure
82-
- other `hashType`s, i. e. `sha1`, `md4`, `md5`, `sha256`, `sha512`
82+
- other `hashType`s, i. e. `xxhash64`, `sha1`, `md4`, `md5`, `sha256`, `sha512`
8383
- other `digestType`s, i. e. `hex`, `base26`, `base32`, `base36`, `base49`, `base52`, `base58`, `base62`, `base64`
8484
- and `length` the length in chars
85-
- `[hash]` the hash of `options.content` (Buffer) (by default it's the hex digest of the md4 hash)
85+
- `[hash]` the hash of `options.content` (Buffer) (by default it's the hex digest of the `xxhash64` hash)
8686
- `[<hashType>:hash:<digestType>:<length>]` optionally one can configure
87-
- other `hashType`s, i. e. `sha1`, `md4`, `md5`, `sha256`, `sha512`
87+
- other `hashType`s, i. e. `xxhash64`, `sha1`, `md4`, `md5`, `sha256`, `sha512`
8888
- other `digestType`s, i. e. `hex`, `base26`, `base32`, `base36`, `base49`, `base52`, `base58`, `base62`, `base64`
8989
- and `length` the length in chars
9090
- `[N]` the N-th match obtained from matching the current file name against `options.regExp`
@@ -118,7 +118,7 @@ loaderUtils.interpolateName(loaderContext, "[hash]", { content: ... });
118118
// loaderContext.resourcePath = "/absolute/path/to/app/img/image.png"
119119
loaderUtils.interpolateName(loaderContext, "[sha512:hash:base64:7].[ext]", { content: ... });
120120
// => 2BKDTjl.png
121-
// use sha512 hash instead of md4 and with only 7 chars of base64
121+
// use sha512 hash instead of xxhash64 and with only 7 chars of base64
122122

123123
// loaderContext.resourcePath = "/absolute/path/to/app/img/myself.png"
124124
// loaderContext.query.name =
@@ -160,7 +160,7 @@ const digestString = loaderUtils.getHashDigest(
160160
```
161161

162162
- `buffer` the content that should be hashed
163-
- `hashType` one of `sha1`, `md4`, `md5`, `sha256`, `sha512` or any other node.js supported hash type
163+
- `hashType` one of `xxhash64`, `sha1`, `md4`, `md5`, `sha256`, `sha512` or any other node.js supported hash type
164164
- `digestType` one of `hex`, `base26`, `base32`, `base36`, `base49`, `base52`, `base58`, `base62`, `base64`
165165
- `maxLength` the maximum length in chars
166166

‎lib/getHashDigest.js

Copy file name to clipboardExpand all lines: lib/getHashDigest.js
+190-6Lines changed: 190 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,168 @@
11
"use strict";
22

3+
// Copied from `webpack`
4+
//#region wasm code: xxhash64 (../../../assembly/hash/xxhash64.asm.ts) --initialMemory 1
5+
const xxhash64 = new WebAssembly.Module(
6+
Buffer.from(
7+
// 1180 bytes
8+
"AGFzbQEAAAABCAJgAX8AYAAAAwQDAQAABQMBAAEGGgV+AUIAC34BQgALfgFCAAt+AUIAC34BQgALByIEBGluaXQAAAZ1cGRhdGUAAQVmaW5hbAACBm1lbW9yeQIACrwIAzAAQtbrgu7q/Yn14AAkAELP1tO+0ser2UIkAUIAJAJC+erQ0OfJoeThACQDQgAkBAvUAQIBfwR+IABFBEAPCyMEIACtfCQEIwAhAiMBIQMjAiEEIwMhBQNAIAIgASkDAELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiECIAMgASkDCELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiEDIAQgASkDEELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiEEIAUgASkDGELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiEFIAAgAUEgaiIBSw0ACyACJAAgAyQBIAQkAiAFJAMLsgYCAX8EfiMEQgBSBH4jACICQgGJIwEiA0IHiXwjAiIEQgyJfCMDIgVCEol8IAJCz9bTvtLHq9lCfkIfiUKHla+vmLbem55/foVCh5Wvr5i23puef35CnaO16oOxjYr6AH0gA0LP1tO+0ser2UJ+Qh+JQoeVr6+Ytt6bnn9+hUKHla+vmLbem55/fkKdo7Xqg7GNivoAfSAEQs/W077Sx6vZQn5CH4lCh5Wvr5i23puef36FQoeVr6+Ytt6bnn9+Qp2jteqDsY2K+gB9IAVCz9bTvtLHq9lCfkIfiUKHla+vmLbem55/foVCh5Wvr5i23puef35CnaO16oOxjYr6AH0FQsXP2bLx5brqJwsjBCAArXx8IQIDQCABQQhqIABNBEAgAiABKQMAQs/W077Sx6vZQn5CH4lCh5Wvr5i23puef36FQhuJQoeVr6+Ytt6bnn9+Qp2jteqDsY2K+gB9IQIgAUEIaiEBDAELCyABQQRqIABNBEACfyACIAE1AgBCh5Wvr5i23puef36FQheJQs/W077Sx6vZQn5C+fPd8Zn2masWfCECIAFBBGoLIQELA0AgACABRwRAIAIgATEAAELFz9my8eW66id+hUILiUKHla+vmLbem55/fiECIAFBAWohAQwBCwtBACACIAJCIYiFQs/W077Sx6vZQn4iAiACQh2IhUL5893xmfaZqxZ+IgIgAkIgiIUiAjcDAEEAIAJCIIgiA0L//wODQiCGIANCgID8/w+DQhCIhCIDQv+BgIDwH4NCEIYgA0KA/oOAgOA/g0IIiIQiA0KPgLyA8IHAB4NCCIYgA0LwgcCHgJ6A+ACDQgSIhCIDQoaMmLDgwIGDBnxCBIhCgYKEiJCgwIABg0InfiADQrDgwIGDhoyYMIR8NwMAQQggAkL/////D4MiAkL//wODQiCGIAJCgID8/w+DQhCIhCICQv+BgIDwH4NCEIYgAkKA/oOAgOA/g0IIiIQiAkKPgLyA8IHAB4NCCIYgAkLwgcCHgJ6A+ACDQgSIhCICQoaMmLDgwIGDBnxCBIhCgYKEiJCgwIABg0InfiACQrDgwIGDhoyYMIR8NwMACw==",
9+
"base64"
10+
)
11+
);
12+
//#endregion
13+
14+
class XxHash64 {
15+
/**
16+
* @param {WebAssembly.Instance} instance wasm instance
17+
*/
18+
constructor(instance) {
19+
const exports = /** @type {any} */ (instance.exports);
20+
21+
exports.init();
22+
23+
this.exports = exports;
24+
this.mem = Buffer.from(exports.memory.buffer, 0, 65536);
25+
this.buffered = 0;
26+
}
27+
28+
reset() {
29+
this.buffered = 0;
30+
this.exports.init();
31+
}
32+
33+
/**
34+
* @param {Buffer | string} data data
35+
* @param {BufferEncoding=} encoding encoding
36+
* @returns {this} itself
37+
*/
38+
update(data, encoding) {
39+
if (typeof data === "string") {
40+
if (data.length < 21845) {
41+
this._updateWithShortString(data, encoding);
42+
43+
return this;
44+
} else {
45+
data = Buffer.from(data, encoding);
46+
}
47+
}
48+
49+
this._updateWithBuffer(data);
50+
51+
return this;
52+
}
53+
54+
/**
55+
* @param {string} data data
56+
* @param {BufferEncoding=} encoding encoding
57+
* @returns {void}
58+
*/
59+
_updateWithShortString(data, encoding) {
60+
const { exports, buffered, mem } = this;
61+
62+
let endPos;
63+
64+
if (data.length < 70) {
65+
if (!encoding || encoding === "utf-8" || encoding === "utf8") {
66+
endPos = buffered;
67+
68+
for (let i = 0; i < data.length; i++) {
69+
const cc = data.charCodeAt(i);
70+
71+
if (cc < 0x80) {
72+
mem[endPos++] = cc;
73+
} else if (cc < 0x800) {
74+
mem[endPos] = (cc >> 6) | 0xc0;
75+
mem[endPos + 1] = (cc & 0x3f) | 0x80;
76+
endPos += 2;
77+
} else {
78+
// bail-out for weird chars
79+
endPos += mem.write(data.slice(endPos), endPos, encoding);
80+
break;
81+
}
82+
}
83+
} else if (encoding === "latin1") {
84+
endPos = buffered;
85+
86+
for (let i = 0; i < data.length; i++) {
87+
const cc = data.charCodeAt(i);
88+
89+
mem[endPos++] = cc;
90+
}
91+
} else {
92+
endPos = buffered + mem.write(data, buffered, encoding);
93+
}
94+
} else {
95+
endPos = buffered + mem.write(data, buffered, encoding);
96+
}
97+
98+
if (endPos < 32) {
99+
this.buffered = endPos;
100+
} else {
101+
const l = (endPos >> 5) << 5;
102+
103+
exports.update(l);
104+
105+
const newBuffered = endPos - l;
106+
107+
this.buffered = newBuffered;
108+
109+
if (newBuffered > 0) {
110+
mem.copyWithin(0, l, endPos);
111+
}
112+
}
113+
}
114+
115+
/**
116+
* @param {Buffer} data data
117+
* @returns {void}
118+
*/
119+
_updateWithBuffer(data) {
120+
const { exports, buffered, mem } = this;
121+
const length = data.length;
122+
if (buffered + length < 32) {
123+
data.copy(mem, buffered, 0, length);
124+
this.buffered += length;
125+
} else {
126+
const l = ((buffered + length) >> 5) << 5;
127+
if (l > 65536) {
128+
let i = 65536 - buffered;
129+
data.copy(mem, buffered, 0, i);
130+
exports.update(65536);
131+
const stop = l - buffered - 65536;
132+
while (i < stop) {
133+
data.copy(mem, 0, i, i + 65536);
134+
exports.update(65536);
135+
i += 65536;
136+
}
137+
data.copy(mem, 0, i, l - buffered);
138+
exports.update(l - buffered - i);
139+
} else {
140+
data.copy(mem, buffered, 0, l - buffered);
141+
exports.update(l);
142+
}
143+
144+
const newBuffered = length + buffered - l;
145+
146+
this.buffered = newBuffered;
147+
148+
if (newBuffered > 0) {
149+
data.copy(mem, 0, length - newBuffered, length);
150+
}
151+
}
152+
}
153+
154+
digest() {
155+
const { exports, buffered, mem } = this;
156+
157+
exports.final(buffered);
158+
instancesPool.push(this);
159+
160+
return mem.toString("latin1", 0, 16);
161+
}
162+
}
163+
164+
const instancesPool = [];
165+
3166
const baseEncodeTables = {
4167
26: "abcdefghijklmnopqrstuvwxyz",
5168
32: "123456789abcdefghjkmnpqrstuvwxyz", // no 0lio
@@ -13,6 +176,7 @@ const baseEncodeTables = {
13176

14177
function encodeBufferToBase(buffer, base) {
15178
const encodeTable = baseEncodeTables[base];
179+
16180
if (!encodeTable) {
17181
throw new Error("Unknown encoding base" + base);
18182
}
@@ -21,13 +185,15 @@ function encodeBufferToBase(buffer, base) {
21185
const Big = require("big.js");
22186

23187
Big.RM = Big.DP = 0;
188+
24189
let b = new Big(0);
25190

26191
for (let i = readLength - 1; i >= 0; i--) {
27192
b = b.times(256).plus(buffer[i]);
28193
}
29194

30195
let output = "";
196+
31197
while (b.gt(0)) {
32198
output = encodeTable[b.mod(base)] + output;
33199
b = b.div(base);
@@ -39,11 +205,29 @@ function encodeBufferToBase(buffer, base) {
39205
return output;
40206
}
41207

208+
const create = () => {
209+
if (instancesPool.length > 0) {
210+
const old = instancesPool.pop();
211+
212+
old.reset();
213+
214+
return old;
215+
} else {
216+
return new XxHash64(new WebAssembly.Instance(xxhash64));
217+
}
218+
};
219+
42220
function getHashDigest(buffer, hashType, digestType, maxLength) {
43-
hashType = hashType || "md4";
221+
hashType = hashType || "xxhash64";
44222
maxLength = maxLength || 9999;
45223

46-
const hash = require("crypto").createHash(hashType);
224+
let hash;
225+
226+
if (hashType === "xxhash64") {
227+
hash = create(maxLength);
228+
} else {
229+
hash = require("crypto").createHash(hashType);
230+
}
47231

48232
hash.update(buffer);
49233

@@ -57,10 +241,10 @@ function getHashDigest(buffer, hashType, digestType, maxLength) {
57241
digestType === "base62" ||
58242
digestType === "base64"
59243
) {
60-
return encodeBufferToBase(hash.digest(), digestType.substr(4)).substr(
61-
0,
62-
maxLength
63-
);
244+
return encodeBufferToBase(
245+
hashType === "xxhash64" ? Buffer.from(hash.digest()) : hash.digest(),
246+
digestType.substr(4)
247+
).substr(0, maxLength);
64248
} else {
65249
return hash.digest(digestType || "hex").substr(0, maxLength);
66250
}

‎test/getHashDigest.test.js

Copy file name to clipboardExpand all lines: test/getHashDigest.test.js
+6-1Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,13 @@ describe("getHashDigest()", () => {
1111
undefined,
1212
"6f8db599de986fab7a21625b7916589c",
1313
],
14-
["test string", "md5", "hex", 4, "6f8d"],
1514
["test string", "md5", "base64", undefined, "2sm1pVmS8xuGJLCdWpJoRL"],
15+
["test string", "md5", "base64url", undefined, "b421md6Yb6t6IWJbeRZYnA"],
16+
["test string", "xxhash64", "hex", undefined, "e9e2c351e3c6b198"],
17+
["test string", "xxhash64", "base64", undefined, "Uej5ydCcPpj4RcScOpjBB"],
18+
["test string", "xxhash64", "base52", undefined, "bqOwublJwrBqLcKHCVpojCL"],
19+
["test string", "xxhash64", "base64url", undefined, "e9e2c351e3c6b198"],
20+
["test string", "md5", "hex", 4, "6f8d"],
1621
["test string", "md5", "base52", undefined, "dJnldHSAutqUacjgfBQGLQx"],
1722
["test string", "md5", "base26", 6, "bhtsgu"],
1823
[

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.