Skip to content

Commit

Permalink
refactor!: use xxhash64 by default for [hash]/[contenthash] and…
Browse files Browse the repository at this point in the history 
… `getHashDigest` API
  • Loading branch information
@alexander-akait
alexander-akait committed Oct 20, 2021
1 parent 10503d6 commit f2ce2ca
Show file tree
Hide file tree
Showing 4 changed files with 222 additions and 43 deletions.
12 changes: 6 additions & 6 deletions README.md
View file
Expand Up @@ -77,14 +77,14 @@ The following tokens are replaced in the `name` parameter:
- `[path]` the path of the resource relative to the `context` query parameter or option.
- `[folder]` the folder the resource is in
- `[query]` the queryof the resource, i.e. `?foo=bar`
- `[contenthash]` the hash of `options.content` (Buffer) (by default it's the hex digest of the md4 hash)
- `[contenthash]` the hash of `options.content` (Buffer) (by default it's the hex digest of the `xxhash64` hash)
- `[<hashType>:contenthash:<digestType>:<length>]` optionally one can configure
- other `hashType`s, i. e. `sha1`, `md4`, `md5`, `sha256`, `sha512`
- other `hashType`s, i. e. `xxhash64`, `sha1`, `md4`, `md5`, `sha256`, `sha512`
- other `digestType`s, i. e. `hex`, `base26`, `base32`, `base36`, `base49`, `base52`, `base58`, `base62`, `base64`
- and `length` the length in chars
- `[hash]` the hash of `options.content` (Buffer) (by default it's the hex digest of the md4 hash)
- `[hash]` the hash of `options.content` (Buffer) (by default it's the hex digest of the `xxhash64` hash)
- `[<hashType>:hash:<digestType>:<length>]` optionally one can configure
- other `hashType`s, i. e. `sha1`, `md4`, `md5`, `sha256`, `sha512`
- other `hashType`s, i. e. `xxhash64`, `sha1`, `md4`, `md5`, `sha256`, `sha512`
- other `digestType`s, i. e. `hex`, `base26`, `base32`, `base36`, `base49`, `base52`, `base58`, `base62`, `base64`
- and `length` the length in chars
- `[N]` the N-th match obtained from matching the current file name against `options.regExp`
Expand Down Expand Up @@ -118,7 +118,7 @@ loaderUtils.interpolateName(loaderContext, "[hash]", { content: ... });
// loaderContext.resourcePath = "/absolute/path/to/app/img/image.png"
loaderUtils.interpolateName(loaderContext, "[sha512:hash:base64:7].[ext]", { content: ... });
// => 2BKDTjl.png
// use sha512 hash instead of md4 and with only 7 chars of base64
// use sha512 hash instead of xxhash64 and with only 7 chars of base64

// loaderContext.resourcePath = "/absolute/path/to/app/img/myself.png"
// loaderContext.query.name =
Expand Down Expand Up @@ -160,7 +160,7 @@ const digestString = loaderUtils.getHashDigest(
```

- `buffer` the content that should be hashed
- `hashType` one of `sha1`, `md4`, `md5`, `sha256`, `sha512` or any other node.js supported hash type
- `hashType` one of `xxhash64`, `sha1`, `md4`, `md5`, `sha256`, `sha512` or any other node.js supported hash type
- `digestType` one of `hex`, `base26`, `base32`, `base36`, `base49`, `base52`, `base58`, `base62`, `base64`
- `maxLength` the maximum length in chars

Expand Down
196 changes: 190 additions & 6 deletions lib/getHashDigest.js
View file
@@ -1,5 +1,168 @@
"use strict";

// Copied from `webpack`
//#region wasm code: xxhash64 (../../../assembly/hash/xxhash64.asm.ts) --initialMemory 1
const xxhash64 = new WebAssembly.Module(
Buffer.from(
// 1180 bytes
"AGFzbQEAAAABCAJgAX8AYAAAAwQDAQAABQMBAAEGGgV+AUIAC34BQgALfgFCAAt+AUIAC34BQgALByIEBGluaXQAAAZ1cGRhdGUAAQVmaW5hbAACBm1lbW9yeQIACrwIAzAAQtbrgu7q/Yn14AAkAELP1tO+0ser2UIkAUIAJAJC+erQ0OfJoeThACQDQgAkBAvUAQIBfwR+IABFBEAPCyMEIACtfCQEIwAhAiMBIQMjAiEEIwMhBQNAIAIgASkDAELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiECIAMgASkDCELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiEDIAQgASkDEELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiEEIAUgASkDGELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiEFIAAgAUEgaiIBSw0ACyACJAAgAyQBIAQkAiAFJAMLsgYCAX8EfiMEQgBSBH4jACICQgGJIwEiA0IHiXwjAiIEQgyJfCMDIgVCEol8IAJCz9bTvtLHq9lCfkIfiUKHla+vmLbem55/foVCh5Wvr5i23puef35CnaO16oOxjYr6AH0gA0LP1tO+0ser2UJ+Qh+JQoeVr6+Ytt6bnn9+hUKHla+vmLbem55/fkKdo7Xqg7GNivoAfSAEQs/W077Sx6vZQn5CH4lCh5Wvr5i23puef36FQoeVr6+Ytt6bnn9+Qp2jteqDsY2K+gB9IAVCz9bTvtLHq9lCfkIfiUKHla+vmLbem55/foVCh5Wvr5i23puef35CnaO16oOxjYr6AH0FQsXP2bLx5brqJwsjBCAArXx8IQIDQCABQQhqIABNBEAgAiABKQMAQs/W077Sx6vZQn5CH4lCh5Wvr5i23puef36FQhuJQoeVr6+Ytt6bnn9+Qp2jteqDsY2K+gB9IQIgAUEIaiEBDAELCyABQQRqIABNBEACfyACIAE1AgBCh5Wvr5i23puef36FQheJQs/W077Sx6vZQn5C+fPd8Zn2masWfCECIAFBBGoLIQELA0AgACABRwRAIAIgATEAAELFz9my8eW66id+hUILiUKHla+vmLbem55/fiECIAFBAWohAQwBCwtBACACIAJCIYiFQs/W077Sx6vZQn4iAiACQh2IhUL5893xmfaZqxZ+IgIgAkIgiIUiAjcDAEEAIAJCIIgiA0L//wODQiCGIANCgID8/w+DQhCIhCIDQv+BgIDwH4NCEIYgA0KA/oOAgOA/g0IIiIQiA0KPgLyA8IHAB4NCCIYgA0LwgcCHgJ6A+ACDQgSIhCIDQoaMmLDgwIGDBnxCBIhCgYKEiJCgwIABg0InfiADQrDgwIGDhoyYMIR8NwMAQQggAkL/////D4MiAkL//wODQiCGIAJCgID8/w+DQhCIhCICQv+BgIDwH4NCEIYgAkKA/oOAgOA/g0IIiIQiAkKPgLyA8IHAB4NCCIYgAkLwgcCHgJ6A+ACDQgSIhCICQoaMmLDgwIGDBnxCBIhCgYKEiJCgwIABg0InfiACQrDgwIGDhoyYMIR8NwMACw==",
"base64"
)
);
//#endregion

class XxHash64 {
/**
* @param {WebAssembly.Instance} instance wasm instance
*/
constructor(instance) {
const exports = /** @type {any} */ (instance.exports);

exports.init();

this.exports = exports;
this.mem = Buffer.from(exports.memory.buffer, 0, 65536);
this.buffered = 0;
}

reset() {
this.buffered = 0;
this.exports.init();
}

/**
* @param {Buffer | string} data data
* @param {BufferEncoding=} encoding encoding
* @returns {this} itself
*/
update(data, encoding) {
if (typeof data === "string") {
if (data.length < 21845) {
this._updateWithShortString(data, encoding);

return this;
} else {
data = Buffer.from(data, encoding);
}
}

this._updateWithBuffer(data);

return this;
}

/**
* @param {string} data data
* @param {BufferEncoding=} encoding encoding
* @returns {void}
*/
_updateWithShortString(data, encoding) {
const { exports, buffered, mem } = this;

let endPos;

if (data.length < 70) {
if (!encoding || encoding === "utf-8" || encoding === "utf8") {
endPos = buffered;

for (let i = 0; i < data.length; i++) {
const cc = data.charCodeAt(i);

if (cc < 0x80) {
mem[endPos++] = cc;
} else if (cc < 0x800) {
mem[endPos] = (cc >> 6) | 0xc0;
mem[endPos + 1] = (cc & 0x3f) | 0x80;
endPos += 2;
} else {
// bail-out for weird chars
endPos += mem.write(data.slice(endPos), endPos, encoding);
break;
}
}
} else if (encoding === "latin1") {
endPos = buffered;

for (let i = 0; i < data.length; i++) {
const cc = data.charCodeAt(i);

mem[endPos++] = cc;
}
} else {
endPos = buffered + mem.write(data, buffered, encoding);
}
} else {
endPos = buffered + mem.write(data, buffered, encoding);
}

if (endPos < 32) {
this.buffered = endPos;
} else {
const l = (endPos >> 5) << 5;

exports.update(l);

const newBuffered = endPos - l;

this.buffered = newBuffered;

if (newBuffered > 0) {
mem.copyWithin(0, l, endPos);
}
}
}

/**
* @param {Buffer} data data
* @returns {void}
*/
_updateWithBuffer(data) {
const { exports, buffered, mem } = this;
const length = data.length;
if (buffered + length < 32) {
data.copy(mem, buffered, 0, length);
this.buffered += length;
} else {
const l = ((buffered + length) >> 5) << 5;
if (l > 65536) {
let i = 65536 - buffered;
data.copy(mem, buffered, 0, i);
exports.update(65536);
const stop = l - buffered - 65536;
while (i < stop) {
data.copy(mem, 0, i, i + 65536);
exports.update(65536);
i += 65536;
}
data.copy(mem, 0, i, l - buffered);
exports.update(l - buffered - i);
} else {
data.copy(mem, buffered, 0, l - buffered);
exports.update(l);
}

const newBuffered = length + buffered - l;

this.buffered = newBuffered;

if (newBuffered > 0) {
data.copy(mem, 0, length - newBuffered, length);
}
}
}

digest() {
const { exports, buffered, mem } = this;

exports.final(buffered);
instancesPool.push(this);

return mem.toString("latin1", 0, 16);
}
}

const instancesPool = [];

const baseEncodeTables = {
26: "abcdefghijklmnopqrstuvwxyz",
32: "123456789abcdefghjkmnpqrstuvwxyz", // no 0lio
Expand All @@ -13,6 +176,7 @@ const baseEncodeTables = {

function encodeBufferToBase(buffer, base) {
const encodeTable = baseEncodeTables[base];

if (!encodeTable) {
throw new Error("Unknown encoding base" + base);
}
Expand All @@ -21,13 +185,15 @@ function encodeBufferToBase(buffer, base) {
const Big = require("big.js");

Big.RM = Big.DP = 0;

let b = new Big(0);

for (let i = readLength - 1; i >= 0; i--) {
b = b.times(256).plus(buffer[i]);
}

let output = "";

while (b.gt(0)) {
output = encodeTable[b.mod(base)] + output;
b = b.div(base);
Expand All @@ -39,11 +205,29 @@ function encodeBufferToBase(buffer, base) {
return output;
}

const create = () => {
if (instancesPool.length > 0) {
const old = instancesPool.pop();

old.reset();

return old;
} else {
return new XxHash64(new WebAssembly.Instance(xxhash64));
}
};

function getHashDigest(buffer, hashType, digestType, maxLength) {
hashType = hashType || "md4";
hashType = hashType || "xxhash64";
maxLength = maxLength || 9999;

const hash = require("crypto").createHash(hashType);
let hash;

if (hashType === "xxhash64") {
hash = create(maxLength);
} else {
hash = require("crypto").createHash(hashType);
}

hash.update(buffer);

Expand All @@ -57,10 +241,10 @@ function getHashDigest(buffer, hashType, digestType, maxLength) {
digestType === "base62" ||
digestType === "base64"
) {
return encodeBufferToBase(hash.digest(), digestType.substr(4)).substr(
0,
maxLength
);
return encodeBufferToBase(
hashType === "xxhash64" ? Buffer.from(hash.digest()) : hash.digest(),
digestType.substr(4)
).substr(0, maxLength);
} else {
return hash.digest(digestType || "hex").substr(0, maxLength);
}
Expand Down
7 changes: 6 additions & 1 deletion test/getHashDigest.test.js
View file
Expand Up @@ -11,8 +11,13 @@ describe("getHashDigest()", () => {
undefined,
"6f8db599de986fab7a21625b7916589c",
],
["test string", "md5", "hex", 4, "6f8d"],
["test string", "md5", "base64", undefined, "2sm1pVmS8xuGJLCdWpJoRL"],
["test string", "md5", "base64url", undefined, "b421md6Yb6t6IWJbeRZYnA"],
["test string", "xxhash64", "hex", undefined, "e9e2c351e3c6b198"],
["test string", "xxhash64", "base64", undefined, "Uej5ydCcPpj4RcScOpjBB"],
["test string", "xxhash64", "base52", undefined, "bqOwublJwrBqLcKHCVpojCL"],
["test string", "xxhash64", "base64url", undefined, "e9e2c351e3c6b198"],
["test string", "md5", "hex", 4, "6f8d"],
["test string", "md5", "base52", undefined, "dJnldHSAutqUacjgfBQGLQx"],
["test string", "md5", "base26", 6, "bhtsgu"],
[
Expand Down

0 comments on commit f2ce2ca

Please sign in to comment.