diff --git a/README.md b/README.md index b261596..b347879 100644 --- a/README.md +++ b/README.md @@ -77,14 +77,14 @@ The following tokens are replaced in the `name` parameter: - `[path]` the path of the resource relative to the `context` query parameter or option. - `[folder]` the folder the resource is in - `[query]` the queryof the resource, i.e. `?foo=bar` -- `[contenthash]` the hash of `options.content` (Buffer) (by default it's the hex digest of the md4 hash) +- `[contenthash]` the hash of `options.content` (Buffer) (by default it's the hex digest of the `xxhash64` hash) - `[:contenthash::]` optionally one can configure - - other `hashType`s, i. e. `sha1`, `md4`, `md5`, `sha256`, `sha512` + - other `hashType`s, i. e. `xxhash64`, `sha1`, `md4`, `md5`, `sha256`, `sha512` - other `digestType`s, i. e. `hex`, `base26`, `base32`, `base36`, `base49`, `base52`, `base58`, `base62`, `base64` - and `length` the length in chars -- `[hash]` the hash of `options.content` (Buffer) (by default it's the hex digest of the md4 hash) +- `[hash]` the hash of `options.content` (Buffer) (by default it's the hex digest of the `xxhash64` hash) - `[:hash::]` optionally one can configure - - other `hashType`s, i. e. `sha1`, `md4`, `md5`, `sha256`, `sha512` + - other `hashType`s, i. e. `xxhash64`, `sha1`, `md4`, `md5`, `sha256`, `sha512` - other `digestType`s, i. e. `hex`, `base26`, `base32`, `base36`, `base49`, `base52`, `base58`, `base62`, `base64` - and `length` the length in chars - `[N]` the N-th match obtained from matching the current file name against `options.regExp` @@ -118,7 +118,7 @@ loaderUtils.interpolateName(loaderContext, "[hash]", { content: ... }); // loaderContext.resourcePath = "/absolute/path/to/app/img/image.png" loaderUtils.interpolateName(loaderContext, "[sha512:hash:base64:7].[ext]", { content: ... }); // => 2BKDTjl.png -// use sha512 hash instead of md4 and with only 7 chars of base64 +// use sha512 hash instead of xxhash64 and with only 7 chars of base64 // loaderContext.resourcePath = "/absolute/path/to/app/img/myself.png" // loaderContext.query.name = @@ -160,7 +160,7 @@ const digestString = loaderUtils.getHashDigest( ``` - `buffer` the content that should be hashed -- `hashType` one of `sha1`, `md4`, `md5`, `sha256`, `sha512` or any other node.js supported hash type +- `hashType` one of `xxhash64`, `sha1`, `md4`, `md5`, `sha256`, `sha512` or any other node.js supported hash type - `digestType` one of `hex`, `base26`, `base32`, `base36`, `base49`, `base52`, `base58`, `base62`, `base64` - `maxLength` the maximum length in chars diff --git a/lib/getHashDigest.js b/lib/getHashDigest.js index 4837c92..a95c4e3 100644 --- a/lib/getHashDigest.js +++ b/lib/getHashDigest.js @@ -1,5 +1,168 @@ "use strict"; +// Copied from `webpack` +//#region wasm code: xxhash64 (../../../assembly/hash/xxhash64.asm.ts) --initialMemory 1 +const xxhash64 = new WebAssembly.Module( + Buffer.from( + // 1180 bytes + "AGFzbQEAAAABCAJgAX8AYAAAAwQDAQAABQMBAAEGGgV+AUIAC34BQgALfgFCAAt+AUIAC34BQgALByIEBGluaXQAAAZ1cGRhdGUAAQVmaW5hbAACBm1lbW9yeQIACrwIAzAAQtbrgu7q/Yn14AAkAELP1tO+0ser2UIkAUIAJAJC+erQ0OfJoeThACQDQgAkBAvUAQIBfwR+IABFBEAPCyMEIACtfCQEIwAhAiMBIQMjAiEEIwMhBQNAIAIgASkDAELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiECIAMgASkDCELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiEDIAQgASkDEELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiEEIAUgASkDGELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiEFIAAgAUEgaiIBSw0ACyACJAAgAyQBIAQkAiAFJAMLsgYCAX8EfiMEQgBSBH4jACICQgGJIwEiA0IHiXwjAiIEQgyJfCMDIgVCEol8IAJCz9bTvtLHq9lCfkIfiUKHla+vmLbem55/foVCh5Wvr5i23puef35CnaO16oOxjYr6AH0gA0LP1tO+0ser2UJ+Qh+JQoeVr6+Ytt6bnn9+hUKHla+vmLbem55/fkKdo7Xqg7GNivoAfSAEQs/W077Sx6vZQn5CH4lCh5Wvr5i23puef36FQoeVr6+Ytt6bnn9+Qp2jteqDsY2K+gB9IAVCz9bTvtLHq9lCfkIfiUKHla+vmLbem55/foVCh5Wvr5i23puef35CnaO16oOxjYr6AH0FQsXP2bLx5brqJwsjBCAArXx8IQIDQCABQQhqIABNBEAgAiABKQMAQs/W077Sx6vZQn5CH4lCh5Wvr5i23puef36FQhuJQoeVr6+Ytt6bnn9+Qp2jteqDsY2K+gB9IQIgAUEIaiEBDAELCyABQQRqIABNBEACfyACIAE1AgBCh5Wvr5i23puef36FQheJQs/W077Sx6vZQn5C+fPd8Zn2masWfCECIAFBBGoLIQELA0AgACABRwRAIAIgATEAAELFz9my8eW66id+hUILiUKHla+vmLbem55/fiECIAFBAWohAQwBCwtBACACIAJCIYiFQs/W077Sx6vZQn4iAiACQh2IhUL5893xmfaZqxZ+IgIgAkIgiIUiAjcDAEEAIAJCIIgiA0L//wODQiCGIANCgID8/w+DQhCIhCIDQv+BgIDwH4NCEIYgA0KA/oOAgOA/g0IIiIQiA0KPgLyA8IHAB4NCCIYgA0LwgcCHgJ6A+ACDQgSIhCIDQoaMmLDgwIGDBnxCBIhCgYKEiJCgwIABg0InfiADQrDgwIGDhoyYMIR8NwMAQQggAkL/////D4MiAkL//wODQiCGIAJCgID8/w+DQhCIhCICQv+BgIDwH4NCEIYgAkKA/oOAgOA/g0IIiIQiAkKPgLyA8IHAB4NCCIYgAkLwgcCHgJ6A+ACDQgSIhCICQoaMmLDgwIGDBnxCBIhCgYKEiJCgwIABg0InfiACQrDgwIGDhoyYMIR8NwMACw==", + "base64" + ) +); +//#endregion + +class XxHash64 { + /** + * @param {WebAssembly.Instance} instance wasm instance + */ + constructor(instance) { + const exports = /** @type {any} */ (instance.exports); + + exports.init(); + + this.exports = exports; + this.mem = Buffer.from(exports.memory.buffer, 0, 65536); + this.buffered = 0; + } + + reset() { + this.buffered = 0; + this.exports.init(); + } + + /** + * @param {Buffer | string} data data + * @param {BufferEncoding=} encoding encoding + * @returns {this} itself + */ + update(data, encoding) { + if (typeof data === "string") { + if (data.length < 21845) { + this._updateWithShortString(data, encoding); + + return this; + } else { + data = Buffer.from(data, encoding); + } + } + + this._updateWithBuffer(data); + + return this; + } + + /** + * @param {string} data data + * @param {BufferEncoding=} encoding encoding + * @returns {void} + */ + _updateWithShortString(data, encoding) { + const { exports, buffered, mem } = this; + + let endPos; + + if (data.length < 70) { + if (!encoding || encoding === "utf-8" || encoding === "utf8") { + endPos = buffered; + + for (let i = 0; i < data.length; i++) { + const cc = data.charCodeAt(i); + + if (cc < 0x80) { + mem[endPos++] = cc; + } else if (cc < 0x800) { + mem[endPos] = (cc >> 6) | 0xc0; + mem[endPos + 1] = (cc & 0x3f) | 0x80; + endPos += 2; + } else { + // bail-out for weird chars + endPos += mem.write(data.slice(endPos), endPos, encoding); + break; + } + } + } else if (encoding === "latin1") { + endPos = buffered; + + for (let i = 0; i < data.length; i++) { + const cc = data.charCodeAt(i); + + mem[endPos++] = cc; + } + } else { + endPos = buffered + mem.write(data, buffered, encoding); + } + } else { + endPos = buffered + mem.write(data, buffered, encoding); + } + + if (endPos < 32) { + this.buffered = endPos; + } else { + const l = (endPos >> 5) << 5; + + exports.update(l); + + const newBuffered = endPos - l; + + this.buffered = newBuffered; + + if (newBuffered > 0) { + mem.copyWithin(0, l, endPos); + } + } + } + + /** + * @param {Buffer} data data + * @returns {void} + */ + _updateWithBuffer(data) { + const { exports, buffered, mem } = this; + const length = data.length; + if (buffered + length < 32) { + data.copy(mem, buffered, 0, length); + this.buffered += length; + } else { + const l = ((buffered + length) >> 5) << 5; + if (l > 65536) { + let i = 65536 - buffered; + data.copy(mem, buffered, 0, i); + exports.update(65536); + const stop = l - buffered - 65536; + while (i < stop) { + data.copy(mem, 0, i, i + 65536); + exports.update(65536); + i += 65536; + } + data.copy(mem, 0, i, l - buffered); + exports.update(l - buffered - i); + } else { + data.copy(mem, buffered, 0, l - buffered); + exports.update(l); + } + + const newBuffered = length + buffered - l; + + this.buffered = newBuffered; + + if (newBuffered > 0) { + data.copy(mem, 0, length - newBuffered, length); + } + } + } + + digest() { + const { exports, buffered, mem } = this; + + exports.final(buffered); + instancesPool.push(this); + + return mem.toString("latin1", 0, 16); + } +} + +const instancesPool = []; + const baseEncodeTables = { 26: "abcdefghijklmnopqrstuvwxyz", 32: "123456789abcdefghjkmnpqrstuvwxyz", // no 0lio @@ -13,6 +176,7 @@ const baseEncodeTables = { function encodeBufferToBase(buffer, base) { const encodeTable = baseEncodeTables[base]; + if (!encodeTable) { throw new Error("Unknown encoding base" + base); } @@ -21,6 +185,7 @@ function encodeBufferToBase(buffer, base) { const Big = require("big.js"); Big.RM = Big.DP = 0; + let b = new Big(0); for (let i = readLength - 1; i >= 0; i--) { @@ -28,6 +193,7 @@ function encodeBufferToBase(buffer, base) { } let output = ""; + while (b.gt(0)) { output = encodeTable[b.mod(base)] + output; b = b.div(base); @@ -39,11 +205,29 @@ function encodeBufferToBase(buffer, base) { return output; } +const create = () => { + if (instancesPool.length > 0) { + const old = instancesPool.pop(); + + old.reset(); + + return old; + } else { + return new XxHash64(new WebAssembly.Instance(xxhash64)); + } +}; + function getHashDigest(buffer, hashType, digestType, maxLength) { - hashType = hashType || "md4"; + hashType = hashType || "xxhash64"; maxLength = maxLength || 9999; - const hash = require("crypto").createHash(hashType); + let hash; + + if (hashType === "xxhash64") { + hash = create(maxLength); + } else { + hash = require("crypto").createHash(hashType); + } hash.update(buffer); @@ -57,10 +241,10 @@ function getHashDigest(buffer, hashType, digestType, maxLength) { digestType === "base62" || digestType === "base64" ) { - return encodeBufferToBase(hash.digest(), digestType.substr(4)).substr( - 0, - maxLength - ); + return encodeBufferToBase( + hashType === "xxhash64" ? Buffer.from(hash.digest()) : hash.digest(), + digestType.substr(4) + ).substr(0, maxLength); } else { return hash.digest(digestType || "hex").substr(0, maxLength); } diff --git a/test/getHashDigest.test.js b/test/getHashDigest.test.js index 471287c..dfe3e42 100644 --- a/test/getHashDigest.test.js +++ b/test/getHashDigest.test.js @@ -11,8 +11,13 @@ describe("getHashDigest()", () => { undefined, "6f8db599de986fab7a21625b7916589c", ], - ["test string", "md5", "hex", 4, "6f8d"], ["test string", "md5", "base64", undefined, "2sm1pVmS8xuGJLCdWpJoRL"], + ["test string", "md5", "base64url", undefined, "b421md6Yb6t6IWJbeRZYnA"], + ["test string", "xxhash64", "hex", undefined, "e9e2c351e3c6b198"], + ["test string", "xxhash64", "base64", undefined, "Uej5ydCcPpj4RcScOpjBB"], + ["test string", "xxhash64", "base52", undefined, "bqOwublJwrBqLcKHCVpojCL"], + ["test string", "xxhash64", "base64url", undefined, "e9e2c351e3c6b198"], + ["test string", "md5", "hex", 4, "6f8d"], ["test string", "md5", "base52", undefined, "dJnldHSAutqUacjgfBQGLQx"], ["test string", "md5", "base26", 6, "bhtsgu"], [ diff --git a/test/interpolateName.test.js b/test/interpolateName.test.js index 84554ff..3e0bbe8 100644 --- a/test/interpolateName.test.js +++ b/test/interpolateName.test.js @@ -24,38 +24,28 @@ describe("interpolateName()", () => { "/app/js/javascript.js", "js/[hash].script.[ext]", "test content", - "js/a69899814931280e2f527219ad6ac754.script.js", + "js/0e6882304e9adbd5.script.js", ], [ "/app/js/javascript.js", "js/[contenthash].script.[ext]", "test content", - "js/a69899814931280e2f527219ad6ac754.script.js", + "js/0e6882304e9adbd5.script.js", ], [ "/app/page.html", "html-[hash:6].html", "test content", - "html-a69899.html", + "html-0e6882.html", ], [ "/app/page.html", "html-[contenthash:6].html", "test content", - "html-a69899.html", - ], - [ - "/app/flash.txt", - "[hash]", - "test content", - "a69899814931280e2f527219ad6ac754", - ], - [ - "/app/flash.txt", - "[contenthash]", - "test content", - "a69899814931280e2f527219ad6ac754", + "html-0e6882.html", ], + ["/app/flash.txt", "[hash]", "test content", "0e6882304e9adbd5"], + ["/app/flash.txt", "[contenthash]", "test content", "0e6882304e9adbd5"], [ "/app/img/image.png", "[sha512:hash:base64:7].[ext]", @@ -72,13 +62,13 @@ describe("interpolateName()", () => { "/app/dir/file.png", "[path][name].[ext]?[hash]", "test content", - "/app/dir/file.png?a69899814931280e2f527219ad6ac754", + "/app/dir/file.png?0e6882304e9adbd5", ], [ "/app/dir/file.png", "[path][name].[ext]?[contenthash]", "test content", - "/app/dir/file.png?a69899814931280e2f527219ad6ac754", + "/app/dir/file.png?0e6882304e9adbd5", ], [ "/vendor/test/images/loading.gif", @@ -133,37 +123,37 @@ describe("interpolateName()", () => { "/app/js/javascript.js?foo=bar", "js/[hash].script.[ext][query]", "test content", - "js/a69899814931280e2f527219ad6ac754.script.js?foo=bar", + "js/0e6882304e9adbd5.script.js?foo=bar", ], [ "/app/js/javascript.js?foo=bar&bar=baz", "js/[hash].script.[ext][query]", "test content", - "js/a69899814931280e2f527219ad6ac754.script.js?foo=bar&bar=baz", + "js/0e6882304e9adbd5.script.js?foo=bar&bar=baz", ], [ "/app/js/javascript.js?foo", "js/[hash].script.[ext][query]", "test content", - "js/a69899814931280e2f527219ad6ac754.script.js?foo", + "js/0e6882304e9adbd5.script.js?foo", ], [ "/app/js/javascript.js?", "js/[hash].script.[ext][query]", "test content", - "js/a69899814931280e2f527219ad6ac754.script.js", + "js/0e6882304e9adbd5.script.js", ], [ "/app/js/javascript.js?a", "js/[hash].script.[ext][query]", "test content", - "js/a69899814931280e2f527219ad6ac754.script.js?a", + "js/0e6882304e9adbd5.script.js?a", ], [ "/app/js/javascript.js?foo=bar#hash", "js/[hash].script.[ext][query]", "test content", - "js/a69899814931280e2f527219ad6ac754.script.js?foo=bar", + "js/0e6882304e9adbd5.script.js?foo=bar", ], [ "/app/js/javascript.js?foo=bar#hash", @@ -174,7 +164,7 @@ describe("interpolateName()", () => { return "js/[hash].script.[ext][query]"; }, "test content", - "js/a69899814931280e2f527219ad6ac754.script.js?foo=bar", + "js/0e6882304e9adbd5.script.js?foo=bar", ], [ "/app/js/javascript.js?a", @@ -185,7 +175,7 @@ describe("interpolateName()", () => { return "js/[hash].script.[ext][query]"; }, "test content", - "js/a69899814931280e2f527219ad6ac754.script.js?a", + "js/0e6882304e9adbd5.script.js?a", ], [ "/app/js/javascript.js", @@ -196,7 +186,7 @@ describe("interpolateName()", () => { return "js/[hash].script.[ext][query]"; }, "test content", - "js/a69899814931280e2f527219ad6ac754.script.js", + "js/0e6882304e9adbd5.script.js", ], [ "/app/js/javascript.js?", @@ -207,7 +197,7 @@ describe("interpolateName()", () => { return "js/[hash].script.[ext][query]"; }, "test content", - "js/a69899814931280e2f527219ad6ac754.script.js", + "js/0e6882304e9adbd5.script.js", ], ].forEach((test) => { it("should interpolate " + test[0] + " " + test[1], () => { @@ -258,12 +248,12 @@ describe("interpolateName()", () => { run([ [ [{}, "", { content: "test string" }], - "2e06edd4f1623268c5a51730d8a0b2af.bin", + "e9e2c351e3c6b198.bin", "should interpolate default tokens", ], [ [{}, "[hash:base64]", { content: "test string" }], - "2LIG3oc1uBNmwOoL7kXgoK", + "Uej5ydCcPpj4RcScOpjBB", "should interpolate [hash] token with options", ], [