From 96bbda0b958cdc6f2da427006b75f66bc5b09170 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Fri, 16 Jun 2023 19:18:13 +0200 Subject: [PATCH 1/9] prelim work on rabin fingerprint --- checksum.js | 28 +++ crypto/gc2-polynomial.js | 405 ++++++++++++++++++++++++++++++++++ crypto/gc2-polynomial.test.js | 106 +++++++++ number.js | 17 ++ test.js | 2 + 5 files changed, 558 insertions(+) create mode 100644 checksum.js create mode 100644 crypto/gc2-polynomial.js create mode 100644 crypto/gc2-polynomial.test.js diff --git a/checksum.js b/checksum.js new file mode 100644 index 0000000..22cd38d --- /dev/null +++ b/checksum.js @@ -0,0 +1,28 @@ +import * as buffer from './buffer.js' + +/** + * Little endian table + * @type {Uint8Array | null} + */ +let _crc32Table = null +const _computeCrc32Table = () => { + if (_crc32Table == null) { + _crc32Table = buffer.createUint8ArrayFromLen(32) + } + let i = 128 + let crc = 1 + do { + if ((crc & 1) > 0) { // @todo this could be optimized + crc = (crc >>> 1) ^ 0x8408 + } else { + crc >>>= 1 + } + for (let j = 0; j < 256; j = j * i) { + _crc32Table[i + j] = crc ^ _crc32Table[j] + } + i >>>= 1 + } while (i > 0) + return _crc32Table +} + +console.log(_computeCrc32Table()) diff --git a/crypto/gc2-polynomial.js b/crypto/gc2-polynomial.js new file mode 100644 index 0000000..39c9564 --- /dev/null +++ b/crypto/gc2-polynomial.js @@ -0,0 +1,405 @@ +import * as math from '../math.js' +import * as webcrypto from 'lib0/webcrypto' +import * as array from '../array.js' +import * as number from '../number.js' +import * as binary from '../binary.js' +import * as buffer from '../buffer.js' + +/** + * This is a GC2 Polynomial abstraction that is not meant for production! + * + * It is easy to understand and it's correctness is as obvious as possible. It can be used to verify + * efficient implementations of algorithms on GC2. + */ +export class GC2Polynomial { + constructor () { + /** + * @type {Set} + */ + this.degrees = new Set() + } +} + +/** + * @param {Uint8Array} bytes + */ +export const createFromBytes = bytes => { + const p = new GC2Polynomial() + for (let bsi = bytes.length - 1, currDegree = 0; bsi >= 0; bsi--) { + const currByte = bytes[bsi] + for (let i = 0; i < 8; i++) { + if (((currByte >>> i) & 1) === 1) { + p.degrees.add(currDegree) + } + currDegree++ + } + } + return p +} + +/** + * Least-significant-byte-first + * + * @param {Uint8Array} bytes + */ +export const createFromBytesLsb = bytes => { + const p = new GC2Polynomial() + for (let bsi = 0, currDegree = 0; bsi < bytes.length; bsi++) { + const currByte = bytes[bsi] + for (let i = 0; i < 8; i++) { + if (((currByte >>> i) & 1) === 1) { + p.degrees.add(currDegree) + } + currDegree++ + } + } + return p +} + +/** + * @param {GC2Polynomial} p + */ +export const toUint8Array = p => { + const max = getHighestDegree(p) + const buf = buffer.createUint8ArrayFromLen(math.floor(max / 8) + 1) + /** + * @param {number} i + */ + const setBit = i => { + const bi = math.floor(i / 8) + buf[buf.length - 1 - bi] |= (1 << (i % 8)) + } + p.degrees.forEach(setBit) + return buf +} + +/** + * @param {GC2Polynomial} p + */ +export const toUint8ArrayLsb = p => { + const max = getHighestDegree(p) + const buf = buffer.createUint8ArrayFromLen(math.floor(max / 8) + 1) + /** + * @param {number} i + */ + const setBit = i => { + const bi = math.floor(i / 8) + buf[bi] |= (1 << (i % 8)) + } + p.degrees.forEach(setBit) + return buf +} + +/** + * Create from unsigned integer (max 32bit uint) - read most-significant-byte first. + * + * @param {number} uint + */ +export const createFromUint = uint => { + const buf = new Uint8Array(4) + for (let i = 0; i < 4; i++) { + buf[i] = uint >>> 8 * (3 - i) + } + return createFromBytes(buf) +} + +/** + * Create a random polynomial of a specified degree. + * + * @param {number} degree + */ +export const createRandom = degree => { + const bs = new Uint8Array(math.floor(degree / 8) + 1) + webcrypto.getRandomValues(bs) + // Get first byte and explicitly set the bit of "degree" to 1 (the result must have the specified + // degree). + const firstByte = bs[0] | 1 << (degree % 8) + // Find out how many bits of the first byte need to be filled with zeros because they are >degree. + const zeros = 7 - (degree % 8) + bs[0] = ((firstByte << zeros) & 0xff) >>> zeros + return createFromBytes(bs) +} + +/** + * @param {GC2Polynomial} p + * @return number + */ +export const getHighestDegree = p => array.fold(array.from(p.degrees), 0, math.max) + +/** + * Add (+) p2 int the p1 polynomial. + * + * Addition is defined as xor in F2. Substraction is equivalent to addition in F2. + * + * @param {GC2Polynomial} p1 + * @param {GC2Polynomial} p2 + */ +export const addInto = (p1, p2) => { + p2.degrees.forEach(degree => { + if (p1.degrees.has(degree)) { + p1.degrees.delete(degree) + } else { + p1.degrees.add(degree) + } + }) +} + +/** + * Or (|) p2 into the p1 polynomial. + * + * Addition is defined as xor in F2. Substraction is equivalent to addition in F2. + * + * @param {GC2Polynomial} p1 + * @param {GC2Polynomial} p2 + */ +export const orInto = (p1, p2) => { + p2.degrees.forEach(degree => { + p1.degrees.add(degree) + }) +} + +/** + * Add (+) p2 to the p1 polynomial. + * + * Addition is defined as xor in F2. Substraction is equivalent to addition in F2. + * + * @param {GC2Polynomial} p1 + * @param {GC2Polynomial} p2 + */ +export const add = (p1, p2) => { + const result = new GC2Polynomial() + p2.degrees.forEach(degree => { + if (!p1.degrees.has(degree)) { + result.degrees.add(degree) + } + }) + p1.degrees.forEach(degree => { + if (!p2.degrees.has(degree)) { + result.degrees.add(degree) + } + }) + return result +} + +/** + * Add (+) p2 to the p1 polynomial. + * + * Addition is defined as xor in F2. Substraction is equivalent to addition in F2. + * + * @param {GC2Polynomial} p + */ +export const clone = (p) => { + const result = new GC2Polynomial() + p.degrees.forEach(d => result.degrees.add(d)) + return result +} + +/** + * Add (+) p2 to the p1 polynomial. + * + * Addition is defined as xor in F2. Substraction is equivalent to addition in F2. + * + * @param {GC2Polynomial} p + * @param {number} degree + */ +export const addDegreeInto = (p, degree) => { + if (p.degrees.has(degree)) { + p.degrees.delete(degree) + } else { + p.degrees.add(degree) + } +} + +/** + * Multiply (•) p1 with p2 and store the result in p1. + * + * @param {GC2Polynomial} p1 + * @param {GC2Polynomial} p2 + */ +export const multiply = (p1, p2) => { + const result = new GC2Polynomial() + p1.degrees.forEach(degree1 => { + p2.degrees.forEach(degree2 => { + addDegreeInto(result, degree1 + degree2) + }) + }) + return result +} + +/** + * Multiply (•) p1 with p2 and store the result in p1. + * + * @param {GC2Polynomial} p + * @param {number} shift + */ +export const shiftLeft = (p, shift) => { + const result = new GC2Polynomial() + p.degrees.forEach(degree => { + const r = degree + shift + r >= 0 && result.degrees.add(r) + }) + return result +} + +/** + * Multiply (•) p1 with p2 and store the result in p1. + * + * @param {GC2Polynomial} p + * @param {number} shift + */ +export const shiftRight = (p, shift) => shiftLeft(p, -shift) + +/** + * Computes p1 % p2. I.e. the remainder of p1/p2. + * + * @param {GC2Polynomial} p1 + * @param {GC2Polynomial} p2 + */ +export const mod = (p1, p2) => { + const maxDeg1 = getHighestDegree(p1) + const maxDeg2 = getHighestDegree(p2) + const result = clone(p1) + for (let i = maxDeg1 - maxDeg2; i >= 0; i--) { + if (result.degrees.has(maxDeg2 + i)) { + const shifted = shiftLeft(p2, i) + addInto(result, shifted) + } + } + return result +} + +/** + * Computes (p^e mod m). + * + * http://en.wikipedia.org/wiki/Modular_exponentiation + * + * @param {GC2Polynomial} p + * @param {number} e + * @param {GC2Polynomial} m + */ +export const modPow = (p, e, m) => { + let result = ONE + while (true) { + if ((e & 1) === 1) { + result = mod(multiply(result, p), m) + } + e >>>= 1 + if (e === 0) { + return result + } + p = mod(multiply(p, p), m) + } +} + +/** + * Find the greatest common divisor using Euclid's Algorithm. + * + * @param {GC2Polynomial} p1 + * @param {GC2Polynomial} p2 + */ +export const gcd = (p1, p2) => { + while (p2.degrees.size > 0) { + const modded = mod(p1, p2) + p1 = p2 + p2 = modded + } + return p1 +} + +/** + * true iff p1 equals p2 + * + * @param {GC2Polynomial} p1 + * @param {GC2Polynomial} p2 + */ +export const equals = (p1, p2) => { + if (p1.degrees.size !== p2.degrees.size) return false + for (const d of p1.degrees) { + if (!p2.degrees.has(d)) return false + } + return true +} + +const X = createFromBytes(new Uint8Array([2])) +const ONE = createFromBytes(new Uint8Array([1])) + +/** + * Computes ( x^(2^p) - x ) mod f + * + * (shamelessly copied from + * https://github.com/opendedup/rabinfingerprint/blob/master/src/org/rabinfingerprint/polynomial/Polynomial.java) + * + * @param {GC2Polynomial} f + * @param {number} p + */ +const reduceExponent = (f, p) => { + // compute (x^q^p mod f) + const q2p = math.pow(2, p) + const x2q2p = modPow(X, q2p, f) + // subtract (x mod f) + return mod(add(x2q2p, X), f) +} + +/** + * BenOr Reducibility Test + * + * Tests and Constructions of Irreducible Polynomials over Finite Fields + * (1997) Shuhong Gao, Daniel Panario + * + * http://citeseer.ist.psu.edu/cache/papers/cs/27167/http:zSzzSzwww.math.clemson.eduzSzfacultyzSzGaozSzpaperszSzGP97a.pdf/gao97tests.pdf + * + * @param {GC2Polynomial} p + */ +export const isIrreducibleBenOr = p => { + const degree = getHighestDegree(p) + for (let i = 1; i < degree / 2; i++) { + const b = reduceExponent(p, i) + const g = gcd(p, b) + if (!equals(g, ONE)) { + return false + } + } + return true +} + +/** + * @param {number} degree + */ +export const createIrreducible = degree => { + while (true) { + const p = createRandom(degree) + if (isIrreducibleBenOr(p)) return p + } +} + +/** + * Create a fingerprint of buf using the irreducible polynomial m. + * + * @param {Uint8Array} buf + * @param {GC2Polynomial} m + */ +export const fingerprint = (buf, m) => toUint8ArrayLsb(mod(createFromBytes(buf), m)) + +export class FingerprintEncoder { + /** + * @param {GC2Polynomial} m The irreducible polynomial + */ + constructor (m) { + this.fingerprint = new GC2Polynomial() + this.m = m + } + + /** + * @param {number} b + */ + write (b) { + const bp = createFromBytes(new Uint8Array([b])) + const fingerprint = shiftLeft(this.fingerprint, 8) + orInto(fingerprint, bp) + this.fingerprint = mod(fingerprint, this.m) + } + + getFingerprint () { + return toUint8ArrayLsb(this.fingerprint) + } +} diff --git a/crypto/gc2-polynomial.test.js b/crypto/gc2-polynomial.test.js new file mode 100644 index 0000000..d7a7ce7 --- /dev/null +++ b/crypto/gc2-polynomial.test.js @@ -0,0 +1,106 @@ +import * as t from '../testing.js' +import * as gc2 from './gc2-polynomial.js' +import * as math from '../math.js' +import * as array from '../array.js' +import * as prng from '../prng.js' +import * as buffer from '../buffer.js' + +/** + * @param {t.TestCase} _tc + */ +export const testPolynomialBasics = _tc => { + const bs = new Uint8Array([1, 11]) + const p = gc2.createFromBytes(bs) + t.assert(p.degrees.has(3)) + t.assert(p.degrees.has(1)) + t.assert(p.degrees.has(0)) + t.assert(p.degrees.has(8)) +} + +/** + * @param {t.TestCase} _tc + */ +export const testIrreducibleInput = _tc => { + const pa = gc2.createFromUint(0x53) + const pb = gc2.createFromUint(0xCA) + const pm = gc2.createFromUint(0x11B) + const px = gc2.multiply(pa, pb) + t.compare(new Uint8Array([0x53]), gc2.toUint8Array(pa)) + t.compare(new Uint8Array([0xCA]), gc2.toUint8Array(pb)) + t.assert(gc2.equals(gc2.createFromUint(0x3F7E), px)) + t.compare(new Uint8Array([0x3F, 0x7E]), gc2.toUint8Array(px)) + const pabm = gc2.mod(px, pm) + t.compare(new Uint8Array([0x1]), gc2.toUint8Array(pabm)) +} + +/** + * @param {t.TestCase} _tc + */ +export const testIrreducibleSpread = _tc => { + const degree = 53 + const N = 200 + const avgSpread = getSpreadAverage(degree, N) + const diffSpread = math.abs(avgSpread - degree) + t.info(`Average spread for degree ${degree} at ${N} repetitions: ${avgSpread}`) + t.assert(diffSpread < 3, 'Spread of irreducible polynomials is within expected range') +} + +/** + * @param {number} degree + * @param {number} tests + */ +const getSpreadAverage = (degree, tests) => { + const spreads = [] + for (let i = 0, test = 0, lastI = 0; test < tests; i++) { + const f = gc2.createRandom(degree) + t.assert(gc2.getHighestDegree(f) === degree) + if (gc2.isIrreducibleBenOr(f)) { + const spread = i - lastI + spreads.push(spread) + lastI = i + test++ + } + } + return array.fold(spreads, 0, math.add) / tests +} + +/** + * @param {t.TestCase} tc + */ +export const testFingerprint = tc => { + /** + * @type {Array} + */ + const dataObjects = [] + const N = 3000 + const K = 53 + const MSIZE = 130 + const irreducible = gc2.createIrreducible(K) + t.info(`N=${N} K=${K} MSIZE=${MSIZE}`) + for (let i = 0; i < N; i++) { + dataObjects.push(prng.uint8Array(tc.prng, MSIZE)) + } + /** + * @type {Array} + */ + let fingerprints1 = [] + t.measureTime('polynomial direct', () => { + fingerprints1 = dataObjects.map((o, _index) => gc2.fingerprint(o, irreducible)) + }) + const testSet = new Set(fingerprints1.map(buffer.toBase64)) + t.assert(testSet.size === N) + /** + * @type {Array} + */ + let fingerprints2 = [] + t.measureTime('polynomial incremental', () => { + fingerprints2 = dataObjects.map((o, _index) => { + const encoder = new gc2.FingerprintEncoder(irreducible) + for (let i = 0; i < o.byteLength; i++) { + encoder.write(o[i]) + } + return encoder.getFingerprint() + }) + }) + t.compare(fingerprints1, fingerprints2) +} diff --git a/number.js b/number.js index 575967e..bbc297a 100644 --- a/number.js +++ b/number.js @@ -18,3 +18,20 @@ export const HIGHEST_UINT32 = binary.BITS32 export const isInteger = Number.isInteger || (num => typeof num === 'number' && isFinite(num) && math.floor(num) === num) export const isNaN = Number.isNaN export const parseInt = Number.parseInt + +/** + * Count the number of "1" bits in an unsigned 32bit number. + * + * Super fun bitcount algorithm by Brian Kernighan. + * + * @param {number} n + */ +export const countBits = n => { + n &= binary.BITS32 + let count = 0 + while (n) { + n &= (n - 1) + count++ + } + return count +} diff --git a/test.js b/test.js index bd32d3c..009589f 100644 --- a/test.js +++ b/test.js @@ -2,6 +2,7 @@ import { runTests } from './testing.js' import * as array from './array.test.js' import * as broadcastchannel from './broadcastchannel.test.js' import * as crypto from './crypto.test.js' +import * as cryptoPolynomial from './crypto/gc2-polynomial.test.js' import * as logging from './logging.test.js' import * as string from './string.test.js' import * as encoding from './encoding.test.js' @@ -44,6 +45,7 @@ runTests({ array, broadcastchannel, crypto, + cryptoPolynomial, logging, string, encoding, From e40edcd571a48b9a1c24540f973dace2f236bd7a Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Tue, 20 Jun 2023 13:14:21 +0200 Subject: [PATCH 2/9] fingerprint compatiblity between implementations --- crypto/gc2-polynomial.js | 83 +++++++++++++++++++++++++++++++++-- crypto/gc2-polynomial.test.js | 26 +++++++++-- 2 files changed, 102 insertions(+), 7 deletions(-) diff --git a/crypto/gc2-polynomial.js b/crypto/gc2-polynomial.js index 39c9564..ad64325 100644 --- a/crypto/gc2-polynomial.js +++ b/crypto/gc2-polynomial.js @@ -1,9 +1,8 @@ import * as math from '../math.js' import * as webcrypto from 'lib0/webcrypto' import * as array from '../array.js' -import * as number from '../number.js' -import * as binary from '../binary.js' import * as buffer from '../buffer.js' +import * as error from '../error.js' /** * This is a GC2 Polynomial abstraction that is not meant for production! @@ -378,7 +377,7 @@ export const createIrreducible = degree => { * @param {Uint8Array} buf * @param {GC2Polynomial} m */ -export const fingerprint = (buf, m) => toUint8ArrayLsb(mod(createFromBytes(buf), m)) +export const fingerprint = (buf, m) => toUint8Array(mod(createFromBytes(buf), m)) export class FingerprintEncoder { /** @@ -400,6 +399,82 @@ export class FingerprintEncoder { } getFingerprint () { - return toUint8ArrayLsb(this.fingerprint) + return toUint8Array(this.fingerprint) + } +} + +/** + * Shift modulo polynomial i bits to the left. Expect that bs[0] === 1. + * + * @param {Uint8Array} bs + * @param {number} lshift + */ +const _shiftBsLeft = (bs, lshift) => { + if (lshift === 0) return bs + bs = new Uint8Array(bs) + bs[0] <<= lshift + for (let i = 1; i < bs.length; i++) { + bs[i - 1] |= bs[i] >>> (8 - lshift) + bs[i] <<= lshift + } + return bs +} + +export class EfficientFingerprintEncoder { + /** + * @param {Uint8Array} m assert(m[0] === 1) + */ + constructor (m) { + this.m = m + this.blen = m.byteLength + this.bs = new Uint8Array(this.blen) + /** + * This describes the position of the most significant byte (starts with 0 and increases with + * shift) + */ + this.bpos = 0 + } + + /** + * Add/Xor/Substract bytes. + * + * Discards bytes that are out of range. + * @todo put this in function or inline + * + * @param {Uint8Array} cs + */ + add (cs) { + const copyLen = math.min(this.blen, cs.byteLength) + // copy from right to left until max is reached + for (let i = 0; i < copyLen; i++) { + this.bs[(this.bpos + this.blen - i - 1) % this.blen] ^= cs[cs.byteLength - i - 1] + } + } + + /** + * @param {number} byte + */ + write (byte) { + // [0,m1,m2,b] + // x <- bpos + // Shift one byte to the left, add b + this.bs[this.bpos] = byte + this.bpos = (this.bpos + 1) % this.blen + // mod + for (let i = 7; i >= 0; i--) { + if (((this.bs[this.bpos] >>> i) & 1) === 1) { + this.add(_shiftBsLeft(this.m, i)) + } + } + if (this.bs[this.bpos] !== 0) { error.unexpectedCase() } + // assert(this.bs[this.bpos] === 0) + } + + getFingerprint () { + const result = new Uint8Array(this.blen - 1) + for (let i = 0; i < result.byteLength; i++) { + result[i] = this.bs[(this.bpos + i + 1) % this.blen] + } + return result } } diff --git a/crypto/gc2-polynomial.test.js b/crypto/gc2-polynomial.test.js index d7a7ce7..c1be09b 100644 --- a/crypto/gc2-polynomial.test.js +++ b/crypto/gc2-polynomial.test.js @@ -72,11 +72,17 @@ export const testFingerprint = tc => { * @type {Array} */ const dataObjects = [] - const N = 3000 - const K = 53 + const N = 1 // 3000 + const K = 32 const MSIZE = 130 - const irreducible = gc2.createIrreducible(K) t.info(`N=${N} K=${K} MSIZE=${MSIZE}`) + /** + * @type {gc2.GC2Polynomial} + */ + let irreducible + t.measureTime(`find irreducible of ${K}`, () => { + irreducible = gc2.createIrreducible(K) + }) for (let i = 0; i < N; i++) { dataObjects.push(prng.uint8Array(tc.prng, MSIZE)) } @@ -103,4 +109,18 @@ export const testFingerprint = tc => { }) }) t.compare(fingerprints1, fingerprints2) + /** + * @type {Array} + */ + let fingerprints3 = [] + t.measureTime('polynomial incremental (efficent))', () => { + fingerprints3 = dataObjects.map((o, _index) => { + const encoder = new gc2.EfficientFingerprintEncoder(gc2.toUint8Array(irreducible)) + for (let i = 0; i < o.byteLength; i++) { + encoder.write(o[i]) + } + return encoder.getFingerprint() + }) + }) + t.compare(fingerprints1, fingerprints3) } From 690cd696c84f637e96a5249ef0d65fdf37a53eea Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Wed, 21 Jun 2023 12:51:15 +0200 Subject: [PATCH 3/9] implemented cached fingerprint encoder --- crypto/gc2-polynomial.js | 117 +++++++++++++++++++++++++++++++--- crypto/gc2-polynomial.test.js | 16 ++++- 2 files changed, 123 insertions(+), 10 deletions(-) diff --git a/crypto/gc2-polynomial.js b/crypto/gc2-polynomial.js index ad64325..1f4c1c9 100644 --- a/crypto/gc2-polynomial.js +++ b/crypto/gc2-polynomial.js @@ -3,6 +3,12 @@ import * as webcrypto from 'lib0/webcrypto' import * as array from '../array.js' import * as buffer from '../buffer.js' import * as error from '../error.js' +import * as map from '../map.js' + +/** + * @param {number} degree + */ +const _degreeToMinByteLength = degree => math.floor(degree / 8) + 1 /** * This is a GC2 Polynomial abstraction that is not meant for production! @@ -57,10 +63,10 @@ export const createFromBytesLsb = bytes => { /** * @param {GC2Polynomial} p + * @param {number} byteLength */ -export const toUint8Array = p => { - const max = getHighestDegree(p) - const buf = buffer.createUint8ArrayFromLen(math.floor(max / 8) + 1) +export const toUint8Array = (p, byteLength = _degreeToMinByteLength(getHighestDegree(p))) => { + const buf = buffer.createUint8ArrayFromLen(byteLength) /** * @param {number} i */ @@ -74,10 +80,10 @@ export const toUint8Array = p => { /** * @param {GC2Polynomial} p + * @param {number} byteLength */ -export const toUint8ArrayLsb = p => { - const max = getHighestDegree(p) - const buf = buffer.createUint8ArrayFromLen(math.floor(max / 8) + 1) +export const toUint8ArrayLsb = (p, byteLength) => { + const buf = buffer.createUint8ArrayFromLen(byteLength) /** * @param {number} i */ @@ -108,7 +114,7 @@ export const createFromUint = uint => { * @param {number} degree */ export const createRandom = degree => { - const bs = new Uint8Array(math.floor(degree / 8) + 1) + const bs = new Uint8Array(_degreeToMinByteLength(degree)) webcrypto.getRandomValues(bs) // Get first byte and explicitly set the bit of "degree" to 1 (the result must have the specified // degree). @@ -377,7 +383,7 @@ export const createIrreducible = degree => { * @param {Uint8Array} buf * @param {GC2Polynomial} m */ -export const fingerprint = (buf, m) => toUint8Array(mod(createFromBytes(buf), m)) +export const fingerprint = (buf, m) => toUint8Array(mod(createFromBytes(buf), m), _degreeToMinByteLength(getHighestDegree(m) - 1)) export class FingerprintEncoder { /** @@ -399,7 +405,7 @@ export class FingerprintEncoder { } getFingerprint () { - return toUint8Array(this.fingerprint) + return toUint8Array(this.fingerprint, _degreeToMinByteLength(getHighestDegree(this.m) - 1)) } } @@ -478,3 +484,96 @@ export class EfficientFingerprintEncoder { return result } } + +/** + * @param {Uint8Array} bs1 + * @param {Uint8Array} bs2 + */ +export const xorBuffers = (bs1, bs2) => { + const res = new Uint8Array(bs1.byteLength) + if (bs1.byteLength !== bs2.byteLength) error.unexpectedCase() + for (let i = 0; i < res.byteLength; i++) { + res[i] = bs1[i] ^ bs2[i] + } + return res +} + +/** + * Maps from a modulo to the precomputed values. + * + * @type {Map} + */ +const _precomputedFingerprintCache = new Map() + +/** + * @param {Uint8Array} m + */ +const ensureCache = m => map.setIfUndefined(_precomputedFingerprintCache, m, () => { + const byteLen = m.byteLength + const cache = new Uint8Array(256 * byteLen) + /** + * @todo not necessary, can be written directly + * @param {number} msb + * @param {Uint8Array} result + */ + const writeCacheResult = (msb, result) => { + for (let i = 0; i < result.byteLength; i++) { + cache[msb * byteLen + i] = result[i] + } + } + writeCacheResult(1, m) // can be written using a native function + // 10101010 + for (let bit = 1; bit < 8; bit++) { + const mBitShifted = _shiftBsLeft(m, bit) + const bitShifted = 1 << bit + for (let j = 0; j < bitShifted; j++) { + // rest is already precomputed + const rest = (bitShifted | j) ^ mBitShifted[0] + // @todo xorBuffers (and creating views) is not necessary + writeCacheResult(bitShifted | j, xorBuffers(cache.slice(rest * byteLen, rest * byteLen + byteLen), mBitShifted)) + if (cache[(bitShifted | j) * byteLen] !== (bitShifted | j)) { error.unexpectedCase() } + } + } + return cache +}) + +export class CachedEfficientFingerprintEncoder { + /** + * @param {Uint8Array} m assert(m[0] === 1) + */ + constructor (m) { + this.m = m + this.blen = m.byteLength + this.bs = new Uint8Array(this.blen) + this.cache = ensureCache(m) + /** + * This describes the position of the most significant byte (starts with 0 and increases with + * shift) + */ + this.bpos = 0 + } + + /** + * @param {number} byte + */ + write (byte) { + // [0,m1,m2,b] + // x <- bpos + // Shift one byte to the left, add b + this.bs[this.bpos] = byte + this.bpos = (this.bpos + 1) % this.blen + const msb = this.bs[this.bpos] + for (let i = 0; i < this.blen; i++) { + this.bs[(this.bpos + i) % this.blen] ^= this.cache[msb * this.blen + i] + } + if (this.bs[this.bpos] !== 0) { error.unexpectedCase() } + } + + getFingerprint () { + const result = new Uint8Array(this.blen - 1) + for (let i = 0; i < result.byteLength; i++) { + result[i] = this.bs[(this.bpos + i + 1) % this.blen] + } + return result + } +} diff --git a/crypto/gc2-polynomial.test.js b/crypto/gc2-polynomial.test.js index c1be09b..f1f4822 100644 --- a/crypto/gc2-polynomial.test.js +++ b/crypto/gc2-polynomial.test.js @@ -72,7 +72,7 @@ export const testFingerprint = tc => { * @type {Array} */ const dataObjects = [] - const N = 1 // 3000 + const N = 3000 const K = 32 const MSIZE = 130 t.info(`N=${N} K=${K} MSIZE=${MSIZE}`) @@ -123,4 +123,18 @@ export const testFingerprint = tc => { }) }) t.compare(fingerprints1, fingerprints3) + /** + * @type {Array} + */ + let fingerprints4 = [] + t.measureTime('polynomial incremental (efficent & cached))', () => { + fingerprints4 = dataObjects.map((o, _index) => { + const encoder = new gc2.CachedEfficientFingerprintEncoder(gc2.toUint8Array(irreducible)) + for (let i = 0; i < o.byteLength; i++) { + encoder.write(o[i]) + } + return encoder.getFingerprint() + }) + }) + t.compare(fingerprints1, fingerprints4) } From df854d3e3a96f7b00a1bbaa0aad140184a88c262 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Wed, 21 Jun 2023 14:33:55 +0200 Subject: [PATCH 4/9] [fingerprint] almost full test coverage --- crypto/gc2-polynomial.js | 95 ++++++++++------------------------- crypto/gc2-polynomial.test.js | 60 +++++++++++++++++++--- number.test.js | 7 ++- 3 files changed, 83 insertions(+), 79 deletions(-) diff --git a/crypto/gc2-polynomial.js b/crypto/gc2-polynomial.js index 1f4c1c9..d208d74 100644 --- a/crypto/gc2-polynomial.js +++ b/crypto/gc2-polynomial.js @@ -2,7 +2,6 @@ import * as math from '../math.js' import * as webcrypto from 'lib0/webcrypto' import * as array from '../array.js' import * as buffer from '../buffer.js' -import * as error from '../error.js' import * as map from '../map.js' /** @@ -26,6 +25,8 @@ export class GC2Polynomial { } /** + * From Uint8Array (MSB). + * * @param {Uint8Array} bytes */ export const createFromBytes = bytes => { @@ -43,25 +44,8 @@ export const createFromBytes = bytes => { } /** - * Least-significant-byte-first + * Transform to Uint8Array (MSB). * - * @param {Uint8Array} bytes - */ -export const createFromBytesLsb = bytes => { - const p = new GC2Polynomial() - for (let bsi = 0, currDegree = 0; bsi < bytes.length; bsi++) { - const currByte = bytes[bsi] - for (let i = 0; i < 8; i++) { - if (((currByte >>> i) & 1) === 1) { - p.degrees.add(currDegree) - } - currDegree++ - } - } - return p -} - -/** * @param {GC2Polynomial} p * @param {number} byteLength */ @@ -78,23 +62,6 @@ export const toUint8Array = (p, byteLength = _degreeToMinByteLength(getHighestDe return buf } -/** - * @param {GC2Polynomial} p - * @param {number} byteLength - */ -export const toUint8ArrayLsb = (p, byteLength) => { - const buf = buffer.createUint8ArrayFromLen(byteLength) - /** - * @param {number} i - */ - const setBit = i => { - const bi = math.floor(i / 8) - buf[bi] |= (1 << (i % 8)) - } - p.degrees.forEach(setBit) - return buf -} - /** * Create from unsigned integer (max 32bit uint) - read most-significant-byte first. * @@ -472,7 +439,7 @@ export class EfficientFingerprintEncoder { this.add(_shiftBsLeft(this.m, i)) } } - if (this.bs[this.bpos] !== 0) { error.unexpectedCase() } + // if (this.bs[this.bpos] !== 0) { error.unexpectedCase() } // assert(this.bs[this.bpos] === 0) } @@ -485,53 +452,33 @@ export class EfficientFingerprintEncoder { } } -/** - * @param {Uint8Array} bs1 - * @param {Uint8Array} bs2 - */ -export const xorBuffers = (bs1, bs2) => { - const res = new Uint8Array(bs1.byteLength) - if (bs1.byteLength !== bs2.byteLength) error.unexpectedCase() - for (let i = 0; i < res.byteLength; i++) { - res[i] = bs1[i] ^ bs2[i] - } - return res -} - /** * Maps from a modulo to the precomputed values. * - * @type {Map} + * @type {Map} */ const _precomputedFingerprintCache = new Map() /** * @param {Uint8Array} m */ -const ensureCache = m => map.setIfUndefined(_precomputedFingerprintCache, m, () => { +const ensureCache = m => map.setIfUndefined(_precomputedFingerprintCache, buffer.toBase64(m), () => { const byteLen = m.byteLength const cache = new Uint8Array(256 * byteLen) - /** - * @todo not necessary, can be written directly - * @param {number} msb - * @param {Uint8Array} result - */ - const writeCacheResult = (msb, result) => { - for (let i = 0; i < result.byteLength; i++) { - cache[msb * byteLen + i] = result[i] - } - } - writeCacheResult(1, m) // can be written using a native function - // 10101010 + // Use dynamic computing to compute the cached results. + // Starting values: cache(0) = 0; cache(1) = m + cache.set(m, byteLen) for (let bit = 1; bit < 8; bit++) { const mBitShifted = _shiftBsLeft(m, bit) const bitShifted = 1 << bit for (let j = 0; j < bitShifted; j++) { // rest is already precomputed - const rest = (bitShifted | j) ^ mBitShifted[0] - // @todo xorBuffers (and creating views) is not necessary - writeCacheResult(bitShifted | j, xorBuffers(cache.slice(rest * byteLen, rest * byteLen + byteLen), mBitShifted)) - if (cache[(bitShifted | j) * byteLen] !== (bitShifted | j)) { error.unexpectedCase() } + const msb = bitShifted | j + const rest = msb ^ mBitShifted[0] + for (let i = 0; i < byteLen; i++) { + cache[msb * byteLen + i] = cache[rest * byteLen + i] ^ mBitShifted[i] + } + // if (cache[(bitShifted | j) * byteLen] !== (bitShifted | j)) { error.unexpectedCase() } } } return cache @@ -566,7 +513,7 @@ export class CachedEfficientFingerprintEncoder { for (let i = 0; i < this.blen; i++) { this.bs[(this.bpos + i) % this.blen] ^= this.cache[msb * this.blen + i] } - if (this.bs[this.bpos] !== 0) { error.unexpectedCase() } + // if (this.bs[this.bpos] !== 0) { error.unexpectedCase() } } getFingerprint () { @@ -577,3 +524,13 @@ export class CachedEfficientFingerprintEncoder { return result } } + +export const StandardIrreducible8 = new Uint8Array([1, 189]) + +export const StandardIrreducible16 = new Uint8Array([1, 244, 157]) + +export const StandardIrreducible32 = new Uint8Array([1, 149, 183, 205, 191]) + +export const StandardIrreducible64 = new Uint8Array([1, 133, 250, 114, 193, 250, 28, 193, 231]) + +export const StandardIrreducible128 = new Uint8Array([1, 94, 109, 166, 228, 6, 222, 102, 239, 27, 128, 184, 13, 50, 112, 169, 199]) diff --git a/crypto/gc2-polynomial.test.js b/crypto/gc2-polynomial.test.js index f1f4822..6ba6e35 100644 --- a/crypto/gc2-polynomial.test.js +++ b/crypto/gc2-polynomial.test.js @@ -37,8 +37,8 @@ export const testIrreducibleInput = _tc => { * @param {t.TestCase} _tc */ export const testIrreducibleSpread = _tc => { - const degree = 53 - const N = 200 + const degree = 32 + const N = 400 const avgSpread = getSpreadAverage(degree, N) const diffSpread = math.abs(avgSpread - degree) t.info(`Average spread for degree ${degree} at ${N} repetitions: ${avgSpread}`) @@ -64,24 +64,57 @@ const getSpreadAverage = (degree, tests) => { return array.fold(spreads, 0, math.add) / tests } +/** + * @param {t.TestCase} _tc + */ +export const testGenerateIrreducibles = _tc => { + /** + * @param {number} byteLen + */ + const testIrreducibleGen = byteLen => { + const K = byteLen * 8 + const irr = gc2.createIrreducible(K) + t.assert(gc2.getHighestDegree(irr) === K, 'degree equals K') + const irrBs = gc2.toUint8Array(irr) + console.log(`K = ${K}`, irrBs) + t.assert(irrBs[0] === 1) + t.assert(irrBs.byteLength === byteLen + 1) + } + testIrreducibleGen(1) + testIrreducibleGen(2) + testIrreducibleGen(4) + testIrreducibleGen(8) + testIrreducibleGen(16) + gc2.isIrreducibleBenOr(gc2.createFromBytes(gc2.StandardIrreducible8)) + gc2.isIrreducibleBenOr(gc2.createFromBytes(gc2.StandardIrreducible16)) + gc2.isIrreducibleBenOr(gc2.createFromBytes(gc2.StandardIrreducible32)) + gc2.isIrreducibleBenOr(gc2.createFromBytes(gc2.StandardIrreducible64)) + gc2.isIrreducibleBenOr(gc2.createFromBytes(gc2.StandardIrreducible128)) +} + /** * @param {t.TestCase} tc + * @param {number} K */ -export const testFingerprint = tc => { +export const _testFingerprintK = (tc, K) => { /** * @type {Array} */ const dataObjects = [] - const N = 3000 - const K = 32 + const N = 300 const MSIZE = 130 t.info(`N=${N} K=${K} MSIZE=${MSIZE}`) /** * @type {gc2.GC2Polynomial} */ let irreducible + /** + * @type {Uint8Array} + */ + let irreducibleBuffer t.measureTime(`find irreducible of ${K}`, () => { irreducible = gc2.createIrreducible(K) + irreducibleBuffer = gc2.toUint8Array(irreducible) }) for (let i = 0; i < N; i++) { dataObjects.push(prng.uint8Array(tc.prng, MSIZE)) @@ -94,7 +127,7 @@ export const testFingerprint = tc => { fingerprints1 = dataObjects.map((o, _index) => gc2.fingerprint(o, irreducible)) }) const testSet = new Set(fingerprints1.map(buffer.toBase64)) - t.assert(testSet.size === N) + t.assert(K < 32 || testSet.size === N) /** * @type {Array} */ @@ -115,7 +148,7 @@ export const testFingerprint = tc => { let fingerprints3 = [] t.measureTime('polynomial incremental (efficent))', () => { fingerprints3 = dataObjects.map((o, _index) => { - const encoder = new gc2.EfficientFingerprintEncoder(gc2.toUint8Array(irreducible)) + const encoder = new gc2.EfficientFingerprintEncoder(irreducibleBuffer) for (let i = 0; i < o.byteLength; i++) { encoder.write(o[i]) } @@ -129,7 +162,7 @@ export const testFingerprint = tc => { let fingerprints4 = [] t.measureTime('polynomial incremental (efficent & cached))', () => { fingerprints4 = dataObjects.map((o, _index) => { - const encoder = new gc2.CachedEfficientFingerprintEncoder(gc2.toUint8Array(irreducible)) + const encoder = new gc2.CachedEfficientFingerprintEncoder(irreducibleBuffer) for (let i = 0; i < o.byteLength; i++) { encoder.write(o[i]) } @@ -138,3 +171,14 @@ export const testFingerprint = tc => { }) t.compare(fingerprints1, fingerprints4) } + +/** + * @param {t.TestCase} tc + */ +export const testFingerprint = tc => { + _testFingerprintK(tc, 8) + _testFingerprintK(tc, 16) + _testFingerprintK(tc, 32) + _testFingerprintK(tc, 64) + _testFingerprintK(tc, 128) +} diff --git a/number.test.js b/number.test.js index 0f6bdd2..bf8c42b 100644 --- a/number.test.js +++ b/number.test.js @@ -4,9 +4,9 @@ import * as random from './random.js' import * as math from './math.js' /** - * @param {t.TestCase} tc + * @param {t.TestCase} _tc */ -export const testNumber = tc => { +export const testNumber = _tc => { t.describe('isNaN') t.assert(number.isNaN(NaN)) t.assert(!number.isNaN(1 / 0)) @@ -18,6 +18,9 @@ export const testNumber = tc => { t.assert(!number.isInteger(NaN)) t.assert(number.isInteger(0)) t.assert(number.isInteger(-1)) + t.assert(number.countBits(1) === 1) + t.assert(number.countBits(3) === 2) + t.assert(number.countBits(128 + 3) === 3) } /** From c72100e88f097cdb04a1c8099949cc1980f039db Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Wed, 21 Jun 2023 14:43:28 +0200 Subject: [PATCH 5/9] [fingerprint] full test coverage --- crypto/gc2-polynomial.js | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/crypto/gc2-polynomial.js b/crypto/gc2-polynomial.js index d208d74..2267880 100644 --- a/crypto/gc2-polynomial.js +++ b/crypto/gc2-polynomial.js @@ -92,6 +92,7 @@ export const createRandom = degree => { return createFromBytes(bs) } + /** * @param {GC2Polynomial} p * @return number @@ -213,14 +214,6 @@ export const shiftLeft = (p, shift) => { return result } -/** - * Multiply (•) p1 with p2 and store the result in p1. - * - * @param {GC2Polynomial} p - * @param {number} shift - */ -export const shiftRight = (p, shift) => shiftLeft(p, -shift) - /** * Computes p1 % p2. I.e. the remainder of p1/p2. * From d2041ca004725ccb6c73434f5ecb9aa7a0cf6b87 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Thu, 22 Jun 2023 01:08:58 +0200 Subject: [PATCH 6/9] restructure hash/rabin --- buffer.js | 17 ++ .../rabin-gf2-polynomial.js | 163 +----------------- hash/rabin-uncached.js | 61 +++++++ hash/rabin.js | 80 +++++++++ .../rabin.test.js | 64 +++---- package.json | 12 +- rollup.config.js | 5 +- test.js | 4 +- tsconfig.json | 2 +- 9 files changed, 208 insertions(+), 200 deletions(-) rename crypto/gc2-polynomial.js => hash/rabin-gf2-polynomial.js (65%) create mode 100644 hash/rabin-uncached.js create mode 100644 hash/rabin.js rename crypto/gc2-polynomial.test.js => hash/rabin.test.js (67%) diff --git a/buffer.js b/buffer.js index 57eded4..3d50186 100644 --- a/buffer.js +++ b/buffer.js @@ -113,3 +113,20 @@ export const encodeAny = data => { * @return {any} */ export const decodeAny = buf => decoding.readAny(decoding.createDecoder(buf)) + +/** + * Shift Byte Array {N} bits to the left. Does not expand byte array. + * + * @param {Uint8Array} bs + * @param {number} N should be in the range of [0-7] + */ +export const shiftNBitsLeft = (bs, N) => { + if (N === 0) return bs + bs = new Uint8Array(bs) + bs[0] <<= N + for (let i = 1; i < bs.length; i++) { + bs[i - 1] |= bs[i] >>> (8 - N) + bs[i] <<= N + } + return bs +} diff --git a/crypto/gc2-polynomial.js b/hash/rabin-gf2-polynomial.js similarity index 65% rename from crypto/gc2-polynomial.js rename to hash/rabin-gf2-polynomial.js index 2267880..5bc90e5 100644 --- a/crypto/gc2-polynomial.js +++ b/hash/rabin-gf2-polynomial.js @@ -2,7 +2,6 @@ import * as math from '../math.js' import * as webcrypto from 'lib0/webcrypto' import * as array from '../array.js' import * as buffer from '../buffer.js' -import * as map from '../map.js' /** * @param {number} degree @@ -92,7 +91,6 @@ export const createRandom = degree => { return createFromBytes(bs) } - /** * @param {GC2Polynomial} p * @return number @@ -345,7 +343,7 @@ export const createIrreducible = degree => { */ export const fingerprint = (buf, m) => toUint8Array(mod(createFromBytes(buf), m), _degreeToMinByteLength(getHighestDegree(m) - 1)) -export class FingerprintEncoder { +export class RabinPolynomialEncoder { /** * @param {GC2Polynomial} m The irreducible polynomial */ @@ -368,162 +366,3 @@ export class FingerprintEncoder { return toUint8Array(this.fingerprint, _degreeToMinByteLength(getHighestDegree(this.m) - 1)) } } - -/** - * Shift modulo polynomial i bits to the left. Expect that bs[0] === 1. - * - * @param {Uint8Array} bs - * @param {number} lshift - */ -const _shiftBsLeft = (bs, lshift) => { - if (lshift === 0) return bs - bs = new Uint8Array(bs) - bs[0] <<= lshift - for (let i = 1; i < bs.length; i++) { - bs[i - 1] |= bs[i] >>> (8 - lshift) - bs[i] <<= lshift - } - return bs -} - -export class EfficientFingerprintEncoder { - /** - * @param {Uint8Array} m assert(m[0] === 1) - */ - constructor (m) { - this.m = m - this.blen = m.byteLength - this.bs = new Uint8Array(this.blen) - /** - * This describes the position of the most significant byte (starts with 0 and increases with - * shift) - */ - this.bpos = 0 - } - - /** - * Add/Xor/Substract bytes. - * - * Discards bytes that are out of range. - * @todo put this in function or inline - * - * @param {Uint8Array} cs - */ - add (cs) { - const copyLen = math.min(this.blen, cs.byteLength) - // copy from right to left until max is reached - for (let i = 0; i < copyLen; i++) { - this.bs[(this.bpos + this.blen - i - 1) % this.blen] ^= cs[cs.byteLength - i - 1] - } - } - - /** - * @param {number} byte - */ - write (byte) { - // [0,m1,m2,b] - // x <- bpos - // Shift one byte to the left, add b - this.bs[this.bpos] = byte - this.bpos = (this.bpos + 1) % this.blen - // mod - for (let i = 7; i >= 0; i--) { - if (((this.bs[this.bpos] >>> i) & 1) === 1) { - this.add(_shiftBsLeft(this.m, i)) - } - } - // if (this.bs[this.bpos] !== 0) { error.unexpectedCase() } - // assert(this.bs[this.bpos] === 0) - } - - getFingerprint () { - const result = new Uint8Array(this.blen - 1) - for (let i = 0; i < result.byteLength; i++) { - result[i] = this.bs[(this.bpos + i + 1) % this.blen] - } - return result - } -} - -/** - * Maps from a modulo to the precomputed values. - * - * @type {Map} - */ -const _precomputedFingerprintCache = new Map() - -/** - * @param {Uint8Array} m - */ -const ensureCache = m => map.setIfUndefined(_precomputedFingerprintCache, buffer.toBase64(m), () => { - const byteLen = m.byteLength - const cache = new Uint8Array(256 * byteLen) - // Use dynamic computing to compute the cached results. - // Starting values: cache(0) = 0; cache(1) = m - cache.set(m, byteLen) - for (let bit = 1; bit < 8; bit++) { - const mBitShifted = _shiftBsLeft(m, bit) - const bitShifted = 1 << bit - for (let j = 0; j < bitShifted; j++) { - // rest is already precomputed - const msb = bitShifted | j - const rest = msb ^ mBitShifted[0] - for (let i = 0; i < byteLen; i++) { - cache[msb * byteLen + i] = cache[rest * byteLen + i] ^ mBitShifted[i] - } - // if (cache[(bitShifted | j) * byteLen] !== (bitShifted | j)) { error.unexpectedCase() } - } - } - return cache -}) - -export class CachedEfficientFingerprintEncoder { - /** - * @param {Uint8Array} m assert(m[0] === 1) - */ - constructor (m) { - this.m = m - this.blen = m.byteLength - this.bs = new Uint8Array(this.blen) - this.cache = ensureCache(m) - /** - * This describes the position of the most significant byte (starts with 0 and increases with - * shift) - */ - this.bpos = 0 - } - - /** - * @param {number} byte - */ - write (byte) { - // [0,m1,m2,b] - // x <- bpos - // Shift one byte to the left, add b - this.bs[this.bpos] = byte - this.bpos = (this.bpos + 1) % this.blen - const msb = this.bs[this.bpos] - for (let i = 0; i < this.blen; i++) { - this.bs[(this.bpos + i) % this.blen] ^= this.cache[msb * this.blen + i] - } - // if (this.bs[this.bpos] !== 0) { error.unexpectedCase() } - } - - getFingerprint () { - const result = new Uint8Array(this.blen - 1) - for (let i = 0; i < result.byteLength; i++) { - result[i] = this.bs[(this.bpos + i + 1) % this.blen] - } - return result - } -} - -export const StandardIrreducible8 = new Uint8Array([1, 189]) - -export const StandardIrreducible16 = new Uint8Array([1, 244, 157]) - -export const StandardIrreducible32 = new Uint8Array([1, 149, 183, 205, 191]) - -export const StandardIrreducible64 = new Uint8Array([1, 133, 250, 114, 193, 250, 28, 193, 231]) - -export const StandardIrreducible128 = new Uint8Array([1, 94, 109, 166, 228, 6, 222, 102, 239, 27, 128, 184, 13, 50, 112, 169, 199]) diff --git a/hash/rabin-uncached.js b/hash/rabin-uncached.js new file mode 100644 index 0000000..11ddd50 --- /dev/null +++ b/hash/rabin-uncached.js @@ -0,0 +1,61 @@ +import * as math from '../math.js' +import * as buffer from '../buffer.js' + +export class RabinUncachedEncoder { + /** + * @param {Uint8Array} m assert(m[0] === 1) + */ + constructor (m) { + this.m = m + this.blen = m.byteLength + this.bs = new Uint8Array(this.blen) + /** + * This describes the position of the most significant byte (starts with 0 and increases with + * shift) + */ + this.bpos = 0 + } + + /** + * Add/Xor/Substract bytes. + * + * Discards bytes that are out of range. + * @todo put this in function or inline + * + * @param {Uint8Array} cs + */ + add (cs) { + const copyLen = math.min(this.blen, cs.byteLength) + // copy from right to left until max is reached + for (let i = 0; i < copyLen; i++) { + this.bs[(this.bpos + this.blen - i - 1) % this.blen] ^= cs[cs.byteLength - i - 1] + } + } + + /** + * @param {number} byte + */ + write (byte) { + // [0,m1,m2,b] + // x <- bpos + // Shift one byte to the left, add b + this.bs[this.bpos] = byte + this.bpos = (this.bpos + 1) % this.blen + // mod + for (let i = 7; i >= 0; i--) { + if (((this.bs[this.bpos] >>> i) & 1) === 1) { + this.add(buffer.shiftNBitsLeft(this.m, i)) + } + } + // if (this.bs[this.bpos] !== 0) { error.unexpectedCase() } + // assert(this.bs[this.bpos] === 0) + } + + getFingerprint () { + const result = new Uint8Array(this.blen - 1) + for (let i = 0; i < result.byteLength; i++) { + result[i] = this.bs[(this.bpos + i + 1) % this.blen] + } + return result + } +} diff --git a/hash/rabin.js b/hash/rabin.js new file mode 100644 index 0000000..e84c349 --- /dev/null +++ b/hash/rabin.js @@ -0,0 +1,80 @@ +import * as buffer from '../buffer.js' +import * as map from '../map.js' + +export const StandardIrreducible8 = new Uint8Array([1, 189]) +export const StandardIrreducible16 = new Uint8Array([1, 244, 157]) +export const StandardIrreducible32 = new Uint8Array([1, 149, 183, 205, 191]) +export const StandardIrreducible64 = new Uint8Array([1, 133, 250, 114, 193, 250, 28, 193, 231]) +export const StandardIrreducible128 = new Uint8Array([1, 94, 109, 166, 228, 6, 222, 102, 239, 27, 128, 184, 13, 50, 112, 169, 199]) + +/** + * Maps from a modulo to the precomputed values. + * + * @type {Map} + */ +const _precomputedFingerprintCache = new Map() + +/** + * @param {Uint8Array} m + */ +const ensureCache = m => map.setIfUndefined(_precomputedFingerprintCache, buffer.toBase64(m), () => { + const byteLen = m.byteLength + const cache = new Uint8Array(256 * byteLen) + // Use dynamic computing to compute the cached results. + // Starting values: cache(0) = 0; cache(1) = m + cache.set(m, byteLen) + for (let bit = 1; bit < 8; bit++) { + const mBitShifted = buffer.shiftNBitsLeft(m, bit) + const bitShifted = 1 << bit + for (let j = 0; j < bitShifted; j++) { + // rest is already precomputed + const msb = bitShifted | j + const rest = msb ^ mBitShifted[0] + for (let i = 0; i < byteLen; i++) { + cache[msb * byteLen + i] = cache[rest * byteLen + i] ^ mBitShifted[i] + } + // if (cache[(bitShifted | j) * byteLen] !== (bitShifted | j)) { error.unexpectedCase() } + } + } + return cache +}) + +export class RabinEncoder { + /** + * @param {Uint8Array} m assert(m[0] === 1) + */ + constructor (m) { + this.m = m + this.blen = m.byteLength + this.bs = new Uint8Array(this.blen) + this.cache = ensureCache(m) + /** + * This describes the position of the most significant byte (starts with 0 and increases with + * shift) + */ + this.bpos = 0 + } + + /** + * @param {number} byte + */ + write (byte) { + // assert(this.bs[0] === 0) + // Shift one byte to the left, add b + this.bs[this.bpos] = byte + this.bpos = (this.bpos + 1) % this.blen + const msb = this.bs[this.bpos] + for (let i = 0; i < this.blen; i++) { + this.bs[(this.bpos + i) % this.blen] ^= this.cache[msb * this.blen + i] + } + // assert(this.bs[this.bpos] !== 0) + } + + getFingerprint () { + const result = new Uint8Array(this.blen - 1) + for (let i = 0; i < result.byteLength; i++) { + result[i] = this.bs[(this.bpos + i + 1) % this.blen] + } + return result + } +} diff --git a/crypto/gc2-polynomial.test.js b/hash/rabin.test.js similarity index 67% rename from crypto/gc2-polynomial.test.js rename to hash/rabin.test.js index 6ba6e35..d802b9a 100644 --- a/crypto/gc2-polynomial.test.js +++ b/hash/rabin.test.js @@ -1,5 +1,7 @@ import * as t from '../testing.js' -import * as gc2 from './gc2-polynomial.js' +import * as gf2 from './rabin-gf2-polynomial.js' +import { RabinUncachedEncoder } from './rabin-uncached.js' +import * as rabin from './rabin.js' import * as math from '../math.js' import * as array from '../array.js' import * as prng from '../prng.js' @@ -10,7 +12,7 @@ import * as buffer from '../buffer.js' */ export const testPolynomialBasics = _tc => { const bs = new Uint8Array([1, 11]) - const p = gc2.createFromBytes(bs) + const p = gf2.createFromBytes(bs) t.assert(p.degrees.has(3)) t.assert(p.degrees.has(1)) t.assert(p.degrees.has(0)) @@ -21,16 +23,16 @@ export const testPolynomialBasics = _tc => { * @param {t.TestCase} _tc */ export const testIrreducibleInput = _tc => { - const pa = gc2.createFromUint(0x53) - const pb = gc2.createFromUint(0xCA) - const pm = gc2.createFromUint(0x11B) - const px = gc2.multiply(pa, pb) - t.compare(new Uint8Array([0x53]), gc2.toUint8Array(pa)) - t.compare(new Uint8Array([0xCA]), gc2.toUint8Array(pb)) - t.assert(gc2.equals(gc2.createFromUint(0x3F7E), px)) - t.compare(new Uint8Array([0x3F, 0x7E]), gc2.toUint8Array(px)) - const pabm = gc2.mod(px, pm) - t.compare(new Uint8Array([0x1]), gc2.toUint8Array(pabm)) + const pa = gf2.createFromUint(0x53) + const pb = gf2.createFromUint(0xCA) + const pm = gf2.createFromUint(0x11B) + const px = gf2.multiply(pa, pb) + t.compare(new Uint8Array([0x53]), gf2.toUint8Array(pa)) + t.compare(new Uint8Array([0xCA]), gf2.toUint8Array(pb)) + t.assert(gf2.equals(gf2.createFromUint(0x3F7E), px)) + t.compare(new Uint8Array([0x3F, 0x7E]), gf2.toUint8Array(px)) + const pabm = gf2.mod(px, pm) + t.compare(new Uint8Array([0x1]), gf2.toUint8Array(pabm)) } /** @@ -52,9 +54,9 @@ export const testIrreducibleSpread = _tc => { const getSpreadAverage = (degree, tests) => { const spreads = [] for (let i = 0, test = 0, lastI = 0; test < tests; i++) { - const f = gc2.createRandom(degree) - t.assert(gc2.getHighestDegree(f) === degree) - if (gc2.isIrreducibleBenOr(f)) { + const f = gf2.createRandom(degree) + t.assert(gf2.getHighestDegree(f) === degree) + if (gf2.isIrreducibleBenOr(f)) { const spread = i - lastI spreads.push(spread) lastI = i @@ -73,9 +75,9 @@ export const testGenerateIrreducibles = _tc => { */ const testIrreducibleGen = byteLen => { const K = byteLen * 8 - const irr = gc2.createIrreducible(K) - t.assert(gc2.getHighestDegree(irr) === K, 'degree equals K') - const irrBs = gc2.toUint8Array(irr) + const irr = gf2.createIrreducible(K) + t.assert(gf2.getHighestDegree(irr) === K, 'degree equals K') + const irrBs = gf2.toUint8Array(irr) console.log(`K = ${K}`, irrBs) t.assert(irrBs[0] === 1) t.assert(irrBs.byteLength === byteLen + 1) @@ -85,18 +87,18 @@ export const testGenerateIrreducibles = _tc => { testIrreducibleGen(4) testIrreducibleGen(8) testIrreducibleGen(16) - gc2.isIrreducibleBenOr(gc2.createFromBytes(gc2.StandardIrreducible8)) - gc2.isIrreducibleBenOr(gc2.createFromBytes(gc2.StandardIrreducible16)) - gc2.isIrreducibleBenOr(gc2.createFromBytes(gc2.StandardIrreducible32)) - gc2.isIrreducibleBenOr(gc2.createFromBytes(gc2.StandardIrreducible64)) - gc2.isIrreducibleBenOr(gc2.createFromBytes(gc2.StandardIrreducible128)) + gf2.isIrreducibleBenOr(gf2.createFromBytes(rabin.StandardIrreducible8)) + gf2.isIrreducibleBenOr(gf2.createFromBytes(rabin.StandardIrreducible16)) + gf2.isIrreducibleBenOr(gf2.createFromBytes(rabin.StandardIrreducible32)) + gf2.isIrreducibleBenOr(gf2.createFromBytes(rabin.StandardIrreducible64)) + gf2.isIrreducibleBenOr(gf2.createFromBytes(rabin.StandardIrreducible128)) } /** * @param {t.TestCase} tc * @param {number} K */ -export const _testFingerprintK = (tc, K) => { +const _testFingerprintK = (tc, K) => { /** * @type {Array} */ @@ -105,7 +107,7 @@ export const _testFingerprintK = (tc, K) => { const MSIZE = 130 t.info(`N=${N} K=${K} MSIZE=${MSIZE}`) /** - * @type {gc2.GC2Polynomial} + * @type {gf2.GC2Polynomial} */ let irreducible /** @@ -113,8 +115,8 @@ export const _testFingerprintK = (tc, K) => { */ let irreducibleBuffer t.measureTime(`find irreducible of ${K}`, () => { - irreducible = gc2.createIrreducible(K) - irreducibleBuffer = gc2.toUint8Array(irreducible) + irreducible = gf2.createIrreducible(K) + irreducibleBuffer = gf2.toUint8Array(irreducible) }) for (let i = 0; i < N; i++) { dataObjects.push(prng.uint8Array(tc.prng, MSIZE)) @@ -124,7 +126,7 @@ export const _testFingerprintK = (tc, K) => { */ let fingerprints1 = [] t.measureTime('polynomial direct', () => { - fingerprints1 = dataObjects.map((o, _index) => gc2.fingerprint(o, irreducible)) + fingerprints1 = dataObjects.map((o, _index) => gf2.fingerprint(o, irreducible)) }) const testSet = new Set(fingerprints1.map(buffer.toBase64)) t.assert(K < 32 || testSet.size === N) @@ -134,7 +136,7 @@ export const _testFingerprintK = (tc, K) => { let fingerprints2 = [] t.measureTime('polynomial incremental', () => { fingerprints2 = dataObjects.map((o, _index) => { - const encoder = new gc2.FingerprintEncoder(irreducible) + const encoder = new gf2.RabinPolynomialEncoder(irreducible) for (let i = 0; i < o.byteLength; i++) { encoder.write(o[i]) } @@ -148,7 +150,7 @@ export const _testFingerprintK = (tc, K) => { let fingerprints3 = [] t.measureTime('polynomial incremental (efficent))', () => { fingerprints3 = dataObjects.map((o, _index) => { - const encoder = new gc2.EfficientFingerprintEncoder(irreducibleBuffer) + const encoder = new RabinUncachedEncoder(irreducibleBuffer) for (let i = 0; i < o.byteLength; i++) { encoder.write(o[i]) } @@ -162,7 +164,7 @@ export const _testFingerprintK = (tc, K) => { let fingerprints4 = [] t.measureTime('polynomial incremental (efficent & cached))', () => { fingerprints4 = dataObjects.map((o, _index) => { - const encoder = new gc2.CachedEfficientFingerprintEncoder(irreducibleBuffer) + const encoder = new rabin.RabinEncoder(irreducibleBuffer) for (let i = 0; i < o.byteLength; i++) { encoder.write(o[i]) } diff --git a/package.json b/package.json index 5570fef..c4be3d7 100644 --- a/package.json +++ b/package.json @@ -83,19 +83,25 @@ "types": "./crypto/aes-gcm.d.ts", "module": "./crypto/aes-gcm.js", "import": "./crypto/aes-gcm.js", - "require": "./dist/crypto/aes-gcm.cjs" + "require": "./dist/aes-gcm.cjs" }, "./crypto/ecdsa": { "types": "./crypto/ecdsa.d.ts", "module": "./crypto/ecdsa.js", "import": "./crypto/ecdsa.js", - "require": "./dist/crypto/ecdsa.cjs" + "require": "./dist/ecdsa.cjs" }, "./crypto/rsa-oaep": { "types": "./crypto/rsa-oaep.d.ts", "module": "./crypto/rsa-oaep.js", "import": "./crypto/rsa-oaep.js", - "require": "./dist/crypto/rsa-oaep.cjs" + "require": "./dist/rsa-oaep.cjs" + }, + "./hash/rabin": { + "types": "./hash/rabin.d.ts", + "module": "./hash/rabin.js", + "import": "./hash/rabin.js", + "require": "./dist/rabin.cjs" }, "./decoding.js": "./decoding.js", "./dist/decoding.cjs": "./dist/decoding.cjs", diff --git a/rollup.config.js b/rollup.config.js index 3fa4f52..6dea38c 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -1,6 +1,9 @@ import * as fs from 'fs' -const files = fs.readdirSync('./').filter(file => /(? fs.readdirSync(root).map(f => root + f)).flat().filter(file => /(? Date: Thu, 22 Jun 2023 01:14:34 +0200 Subject: [PATCH 7/9] add description to rabin modules --- hash/rabin-gf2-polynomial.js | 11 +++++++++++ hash/rabin-uncached.js | 7 +++++++ hash/rabin.js | 7 +++++++ 3 files changed, 25 insertions(+) diff --git a/hash/rabin-gf2-polynomial.js b/hash/rabin-gf2-polynomial.js index 5bc90e5..8579edf 100644 --- a/hash/rabin-gf2-polynomial.js +++ b/hash/rabin-gf2-polynomial.js @@ -1,3 +1,14 @@ +/** + * The idea of the Rabin fingerprint algorithm is to represent the binary as a polynomial in a + * finite field (Galois Field G(2)). The polynomial will then be taken "modulo" by an irreducible + * polynomial of the desired size. + * + * This implementation is inefficient and is solely used to verify the actually performant + * implementation in `./rabin.js`. + * + * @module rabin-gf2-polynomial + */ + import * as math from '../math.js' import * as webcrypto from 'lib0/webcrypto' import * as array from '../array.js' diff --git a/hash/rabin-uncached.js b/hash/rabin-uncached.js index 11ddd50..f40da97 100644 --- a/hash/rabin-uncached.js +++ b/hash/rabin-uncached.js @@ -1,3 +1,10 @@ +/** + * It is not recommended to use this package. This is the uncached implementation of the rabin + * fingerprint algorithm. However, it can be used to verify the `rabin.js` implementation. + * + * @module rabin-uncached + */ + import * as math from '../math.js' import * as buffer from '../buffer.js' diff --git a/hash/rabin.js b/hash/rabin.js index e84c349..9526fd3 100644 --- a/hash/rabin.js +++ b/hash/rabin.js @@ -1,3 +1,10 @@ +/** + * @module rabin + * + * Very efficient & versatile fingerprint/hashing algorithm. However, it is not cryptographically + * secure. + */ + import * as buffer from '../buffer.js' import * as map from '../map.js' From b7915f3e363665fb43deaf8482175e1505c735e1 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Thu, 22 Jun 2023 16:44:11 +0200 Subject: [PATCH 8/9] [rabin] more tests & documentation --- hash/rabin-gf2-polynomial.js | 68 ++++++++++++++++++------------------ hash/rabin.js | 16 ++++++++- hash/rabin.test.js | 66 ++++++++++++++++++++++++++++------ 3 files changed, 105 insertions(+), 45 deletions(-) diff --git a/hash/rabin-gf2-polynomial.js b/hash/rabin-gf2-polynomial.js index 8579edf..6ae87b2 100644 --- a/hash/rabin-gf2-polynomial.js +++ b/hash/rabin-gf2-polynomial.js @@ -20,12 +20,12 @@ import * as buffer from '../buffer.js' const _degreeToMinByteLength = degree => math.floor(degree / 8) + 1 /** - * This is a GC2 Polynomial abstraction that is not meant for production! + * This is a GF2 Polynomial abstraction that is not meant for production! * * It is easy to understand and it's correctness is as obvious as possible. It can be used to verify - * efficient implementations of algorithms on GC2. + * efficient implementations of algorithms on GF2. */ -export class GC2Polynomial { +export class GF2Polynomial { constructor () { /** * @type {Set} @@ -40,7 +40,7 @@ export class GC2Polynomial { * @param {Uint8Array} bytes */ export const createFromBytes = bytes => { - const p = new GC2Polynomial() + const p = new GF2Polynomial() for (let bsi = bytes.length - 1, currDegree = 0; bsi >= 0; bsi--) { const currByte = bytes[bsi] for (let i = 0; i < 8; i++) { @@ -56,7 +56,7 @@ export const createFromBytes = bytes => { /** * Transform to Uint8Array (MSB). * - * @param {GC2Polynomial} p + * @param {GF2Polynomial} p * @param {number} byteLength */ export const toUint8Array = (p, byteLength = _degreeToMinByteLength(getHighestDegree(p))) => { @@ -103,7 +103,7 @@ export const createRandom = degree => { } /** - * @param {GC2Polynomial} p + * @param {GF2Polynomial} p * @return number */ export const getHighestDegree = p => array.fold(array.from(p.degrees), 0, math.max) @@ -113,8 +113,8 @@ export const getHighestDegree = p => array.fold(array.from(p.degrees), 0, math.m * * Addition is defined as xor in F2. Substraction is equivalent to addition in F2. * - * @param {GC2Polynomial} p1 - * @param {GC2Polynomial} p2 + * @param {GF2Polynomial} p1 + * @param {GF2Polynomial} p2 */ export const addInto = (p1, p2) => { p2.degrees.forEach(degree => { @@ -131,8 +131,8 @@ export const addInto = (p1, p2) => { * * Addition is defined as xor in F2. Substraction is equivalent to addition in F2. * - * @param {GC2Polynomial} p1 - * @param {GC2Polynomial} p2 + * @param {GF2Polynomial} p1 + * @param {GF2Polynomial} p2 */ export const orInto = (p1, p2) => { p2.degrees.forEach(degree => { @@ -145,11 +145,11 @@ export const orInto = (p1, p2) => { * * Addition is defined as xor in F2. Substraction is equivalent to addition in F2. * - * @param {GC2Polynomial} p1 - * @param {GC2Polynomial} p2 + * @param {GF2Polynomial} p1 + * @param {GF2Polynomial} p2 */ export const add = (p1, p2) => { - const result = new GC2Polynomial() + const result = new GF2Polynomial() p2.degrees.forEach(degree => { if (!p1.degrees.has(degree)) { result.degrees.add(degree) @@ -168,10 +168,10 @@ export const add = (p1, p2) => { * * Addition is defined as xor in F2. Substraction is equivalent to addition in F2. * - * @param {GC2Polynomial} p + * @param {GF2Polynomial} p */ export const clone = (p) => { - const result = new GC2Polynomial() + const result = new GF2Polynomial() p.degrees.forEach(d => result.degrees.add(d)) return result } @@ -181,7 +181,7 @@ export const clone = (p) => { * * Addition is defined as xor in F2. Substraction is equivalent to addition in F2. * - * @param {GC2Polynomial} p + * @param {GF2Polynomial} p * @param {number} degree */ export const addDegreeInto = (p, degree) => { @@ -195,11 +195,11 @@ export const addDegreeInto = (p, degree) => { /** * Multiply (•) p1 with p2 and store the result in p1. * - * @param {GC2Polynomial} p1 - * @param {GC2Polynomial} p2 + * @param {GF2Polynomial} p1 + * @param {GF2Polynomial} p2 */ export const multiply = (p1, p2) => { - const result = new GC2Polynomial() + const result = new GF2Polynomial() p1.degrees.forEach(degree1 => { p2.degrees.forEach(degree2 => { addDegreeInto(result, degree1 + degree2) @@ -211,11 +211,11 @@ export const multiply = (p1, p2) => { /** * Multiply (•) p1 with p2 and store the result in p1. * - * @param {GC2Polynomial} p + * @param {GF2Polynomial} p * @param {number} shift */ export const shiftLeft = (p, shift) => { - const result = new GC2Polynomial() + const result = new GF2Polynomial() p.degrees.forEach(degree => { const r = degree + shift r >= 0 && result.degrees.add(r) @@ -226,8 +226,8 @@ export const shiftLeft = (p, shift) => { /** * Computes p1 % p2. I.e. the remainder of p1/p2. * - * @param {GC2Polynomial} p1 - * @param {GC2Polynomial} p2 + * @param {GF2Polynomial} p1 + * @param {GF2Polynomial} p2 */ export const mod = (p1, p2) => { const maxDeg1 = getHighestDegree(p1) @@ -247,9 +247,9 @@ export const mod = (p1, p2) => { * * http://en.wikipedia.org/wiki/Modular_exponentiation * - * @param {GC2Polynomial} p + * @param {GF2Polynomial} p * @param {number} e - * @param {GC2Polynomial} m + * @param {GF2Polynomial} m */ export const modPow = (p, e, m) => { let result = ONE @@ -268,8 +268,8 @@ export const modPow = (p, e, m) => { /** * Find the greatest common divisor using Euclid's Algorithm. * - * @param {GC2Polynomial} p1 - * @param {GC2Polynomial} p2 + * @param {GF2Polynomial} p1 + * @param {GF2Polynomial} p2 */ export const gcd = (p1, p2) => { while (p2.degrees.size > 0) { @@ -283,8 +283,8 @@ export const gcd = (p1, p2) => { /** * true iff p1 equals p2 * - * @param {GC2Polynomial} p1 - * @param {GC2Polynomial} p2 + * @param {GF2Polynomial} p1 + * @param {GF2Polynomial} p2 */ export const equals = (p1, p2) => { if (p1.degrees.size !== p2.degrees.size) return false @@ -303,7 +303,7 @@ const ONE = createFromBytes(new Uint8Array([1])) * (shamelessly copied from * https://github.com/opendedup/rabinfingerprint/blob/master/src/org/rabinfingerprint/polynomial/Polynomial.java) * - * @param {GC2Polynomial} f + * @param {GF2Polynomial} f * @param {number} p */ const reduceExponent = (f, p) => { @@ -322,7 +322,7 @@ const reduceExponent = (f, p) => { * * http://citeseer.ist.psu.edu/cache/papers/cs/27167/http:zSzzSzwww.math.clemson.eduzSzfacultyzSzGaozSzpaperszSzGP97a.pdf/gao97tests.pdf * - * @param {GC2Polynomial} p + * @param {GF2Polynomial} p */ export const isIrreducibleBenOr = p => { const degree = getHighestDegree(p) @@ -350,16 +350,16 @@ export const createIrreducible = degree => { * Create a fingerprint of buf using the irreducible polynomial m. * * @param {Uint8Array} buf - * @param {GC2Polynomial} m + * @param {GF2Polynomial} m */ export const fingerprint = (buf, m) => toUint8Array(mod(createFromBytes(buf), m), _degreeToMinByteLength(getHighestDegree(m) - 1)) export class RabinPolynomialEncoder { /** - * @param {GC2Polynomial} m The irreducible polynomial + * @param {GF2Polynomial} m The irreducible polynomial */ constructor (m) { - this.fingerprint = new GC2Polynomial() + this.fingerprint = new GF2Polynomial() this.m = m } diff --git a/hash/rabin.js b/hash/rabin.js index 9526fd3..90f0a28 100644 --- a/hash/rabin.js +++ b/hash/rabin.js @@ -8,7 +8,7 @@ import * as buffer from '../buffer.js' import * as map from '../map.js' -export const StandardIrreducible8 = new Uint8Array([1, 189]) +export const StandardIrreducible8 = new Uint8Array([1, 221]) export const StandardIrreducible16 = new Uint8Array([1, 244, 157]) export const StandardIrreducible32 = new Uint8Array([1, 149, 183, 205, 191]) export const StandardIrreducible64 = new Uint8Array([1, 133, 250, 114, 193, 250, 28, 193, 231]) @@ -85,3 +85,17 @@ export class RabinEncoder { return result } } + +/** + * Basically an exact copy of the Encoder, but inlined. + * + * @param {Uint8Array} irreducible + * @param {Uint8Array} data + */ +export const fingerprint = (irreducible, data) => { + const encoder = new RabinEncoder(irreducible) + for (let i = 0; i < data.length; i++) { + encoder.write(data[i]) + } + return encoder.getFingerprint() +} diff --git a/hash/rabin.test.js b/hash/rabin.test.js index d802b9a..37a82ff 100644 --- a/hash/rabin.test.js +++ b/hash/rabin.test.js @@ -6,6 +6,7 @@ import * as math from '../math.js' import * as array from '../array.js' import * as prng from '../prng.js' import * as buffer from '../buffer.js' +import * as map from '../map.js' /** * @param {t.TestCase} _tc @@ -40,7 +41,7 @@ export const testIrreducibleInput = _tc => { */ export const testIrreducibleSpread = _tc => { const degree = 32 - const N = 400 + const N = 1000 const avgSpread = getSpreadAverage(degree, N) const diffSpread = math.abs(avgSpread - degree) t.info(`Average spread for degree ${degree} at ${N} repetitions: ${avgSpread}`) @@ -98,7 +99,7 @@ export const testGenerateIrreducibles = _tc => { * @param {t.TestCase} tc * @param {number} K */ -const _testFingerprintK = (tc, K) => { +const _testFingerprintCompatiblityK = (tc, K) => { /** * @type {Array} */ @@ -107,7 +108,7 @@ const _testFingerprintK = (tc, K) => { const MSIZE = 130 t.info(`N=${N} K=${K} MSIZE=${MSIZE}`) /** - * @type {gf2.GC2Polynomial} + * @type {gf2.GF2Polynomial} */ let irreducible /** @@ -158,11 +159,15 @@ const _testFingerprintK = (tc, K) => { }) }) t.compare(fingerprints1, fingerprints3) + // ensuring that the cache is already populated + // @ts-ignore + // eslint-disable-next-line + new rabin.RabinEncoder(irreducibleBuffer) /** * @type {Array} */ let fingerprints4 = [] - t.measureTime('polynomial incremental (efficent & cached))', () => { + t.measureTime('polynomial incremental (efficent & cached)) using encoder', () => { fingerprints4 = dataObjects.map((o, _index) => { const encoder = new rabin.RabinEncoder(irreducibleBuffer) for (let i = 0; i < o.byteLength; i++) { @@ -172,15 +177,56 @@ const _testFingerprintK = (tc, K) => { }) }) t.compare(fingerprints1, fingerprints4) + /** + * @type {Array} + */ + let fingerprints5 = [] + t.measureTime('polynomial incremental (efficent & cached))', () => { + fingerprints5 = dataObjects.map((o, _index) => { + return rabin.fingerprint(irreducibleBuffer, o) + }) + }) + t.compare(fingerprints1, fingerprints5) } /** * @param {t.TestCase} tc */ -export const testFingerprint = tc => { - _testFingerprintK(tc, 8) - _testFingerprintK(tc, 16) - _testFingerprintK(tc, 32) - _testFingerprintK(tc, 64) - _testFingerprintK(tc, 128) +export const testFingerprintCompatiblity = tc => { + _testFingerprintCompatiblityK(tc, 8) + _testFingerprintCompatiblityK(tc, 16) + _testFingerprintCompatiblityK(tc, 32) + _testFingerprintCompatiblityK(tc, 64) + _testFingerprintCompatiblityK(tc, 128) +} + +/** + * @param {t.TestCase} tc + */ +export const testConflicts = tc => { + /** + * @type {Array} + */ + const data = [] + const N = 100 + const Irr = rabin.StandardIrreducible8 + t.measureTime(`generate ${N} items`, () => { + for (let i = 0; i < N; i++) { + data.push(prng.uint8Array(tc.prng, prng.uint32(tc.prng, 5, 50))) + } + }) + /** + * @type {Map>} + */ + const results = new Map() + t.measureTime(`fingerprint ${N} items`, () => { + data.forEach(d => { + const f = buffer.toBase64(rabin.fingerprint(Irr, d)) + map.setIfUndefined(results, f, () => new Set()).add(buffer.toBase64(d)) + }) + }) + const conflicts = array.fold(map.map(results, (ds) => ds.size - 1), 0, math.add) + const usedFields = results.size + const unusedFieds = math.pow(2, (Irr.length - 1) * 8) - results.size + console.log({ conflicts, usedFields, unusedFieds }) } From ea4f18581054d949a109e54dd85c68a29ab6873d Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Thu, 22 Jun 2023 17:14:11 +0200 Subject: [PATCH 9/9] [rabin] update comments --- hash/rabin.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/hash/rabin.js b/hash/rabin.js index 90f0a28..d5690f1 100644 --- a/hash/rabin.js +++ b/hash/rabin.js @@ -2,7 +2,7 @@ * @module rabin * * Very efficient & versatile fingerprint/hashing algorithm. However, it is not cryptographically - * secure. + * secure. Well suited for fingerprinting. */ import * as buffer from '../buffer.js' @@ -34,10 +34,11 @@ const ensureCache = m => map.setIfUndefined(_precomputedFingerprintCache, buffer const mBitShifted = buffer.shiftNBitsLeft(m, bit) const bitShifted = 1 << bit for (let j = 0; j < bitShifted; j++) { - // rest is already precomputed + // apply the shifted result (reducing the degree of the polynomial) const msb = bitShifted | j const rest = msb ^ mBitShifted[0] for (let i = 0; i < byteLen; i++) { + // rest is already precomputed in the cache cache[msb * byteLen + i] = cache[rest * byteLen + i] ^ mBitShifted[i] } // if (cache[(bitShifted | j) * byteLen] !== (bitShifted | j)) { error.unexpectedCase() } @@ -74,7 +75,7 @@ export class RabinEncoder { for (let i = 0; i < this.blen; i++) { this.bs[(this.bpos + i) % this.blen] ^= this.cache[msb * this.blen + i] } - // assert(this.bs[this.bpos] !== 0) + // assert(this.bs[this.bpos] === 0) } getFingerprint () { @@ -87,8 +88,6 @@ export class RabinEncoder { } /** - * Basically an exact copy of the Encoder, but inlined. - * * @param {Uint8Array} irreducible * @param {Uint8Array} data */