diff --git a/checksum.js b/checksum.js new file mode 100644 index 0000000..22cd38d --- /dev/null +++ b/checksum.js @@ -0,0 +1,28 @@ +import * as buffer from './buffer.js' + +/** + * Little endian table + * @type {Uint8Array | null} + */ +let _crc32Table = null +const _computeCrc32Table = () => { + if (_crc32Table == null) { + _crc32Table = buffer.createUint8ArrayFromLen(32) + } + let i = 128 + let crc = 1 + do { + if ((crc & 1) > 0) { // @todo this could be optimized + crc = (crc >>> 1) ^ 0x8408 + } else { + crc >>>= 1 + } + for (let j = 0; j < 256; j = j * i) { + _crc32Table[i + j] = crc ^ _crc32Table[j] + } + i >>>= 1 + } while (i > 0) + return _crc32Table +} + +console.log(_computeCrc32Table()) diff --git a/crypto/gc2-polynomial.js b/crypto/gc2-polynomial.js new file mode 100644 index 0000000..39c9564 --- /dev/null +++ b/crypto/gc2-polynomial.js @@ -0,0 +1,405 @@ +import * as math from '../math.js' +import * as webcrypto from 'lib0/webcrypto' +import * as array from '../array.js' +import * as number from '../number.js' +import * as binary from '../binary.js' +import * as buffer from '../buffer.js' + +/** + * This is a GC2 Polynomial abstraction that is not meant for production! + * + * It is easy to understand and it's correctness is as obvious as possible. It can be used to verify + * efficient implementations of algorithms on GC2. + */ +export class GC2Polynomial { + constructor () { + /** + * @type {Set} + */ + this.degrees = new Set() + } +} + +/** + * @param {Uint8Array} bytes + */ +export const createFromBytes = bytes => { + const p = new GC2Polynomial() + for (let bsi = bytes.length - 1, currDegree = 0; bsi >= 0; bsi--) { + const currByte = bytes[bsi] + for (let i = 0; i < 8; i++) { + if (((currByte >>> i) & 1) === 1) { + p.degrees.add(currDegree) + } + currDegree++ + } + } + return p +} + +/** + * Least-significant-byte-first + * + * @param {Uint8Array} bytes + */ +export const createFromBytesLsb = bytes => { + const p = new GC2Polynomial() + for (let bsi = 0, currDegree = 0; bsi < bytes.length; bsi++) { + const currByte = bytes[bsi] + for (let i = 0; i < 8; i++) { + if (((currByte >>> i) & 1) === 1) { + p.degrees.add(currDegree) + } + currDegree++ + } + } + return p +} + +/** + * @param {GC2Polynomial} p + */ +export const toUint8Array = p => { + const max = getHighestDegree(p) + const buf = buffer.createUint8ArrayFromLen(math.floor(max / 8) + 1) + /** + * @param {number} i + */ + const setBit = i => { + const bi = math.floor(i / 8) + buf[buf.length - 1 - bi] |= (1 << (i % 8)) + } + p.degrees.forEach(setBit) + return buf +} + +/** + * @param {GC2Polynomial} p + */ +export const toUint8ArrayLsb = p => { + const max = getHighestDegree(p) + const buf = buffer.createUint8ArrayFromLen(math.floor(max / 8) + 1) + /** + * @param {number} i + */ + const setBit = i => { + const bi = math.floor(i / 8) + buf[bi] |= (1 << (i % 8)) + } + p.degrees.forEach(setBit) + return buf +} + +/** + * Create from unsigned integer (max 32bit uint) - read most-significant-byte first. + * + * @param {number} uint + */ +export const createFromUint = uint => { + const buf = new Uint8Array(4) + for (let i = 0; i < 4; i++) { + buf[i] = uint >>> 8 * (3 - i) + } + return createFromBytes(buf) +} + +/** + * Create a random polynomial of a specified degree. + * + * @param {number} degree + */ +export const createRandom = degree => { + const bs = new Uint8Array(math.floor(degree / 8) + 1) + webcrypto.getRandomValues(bs) + // Get first byte and explicitly set the bit of "degree" to 1 (the result must have the specified + // degree). + const firstByte = bs[0] | 1 << (degree % 8) + // Find out how many bits of the first byte need to be filled with zeros because they are >degree. + const zeros = 7 - (degree % 8) + bs[0] = ((firstByte << zeros) & 0xff) >>> zeros + return createFromBytes(bs) +} + +/** + * @param {GC2Polynomial} p + * @return number + */ +export const getHighestDegree = p => array.fold(array.from(p.degrees), 0, math.max) + +/** + * Add (+) p2 int the p1 polynomial. + * + * Addition is defined as xor in F2. Substraction is equivalent to addition in F2. + * + * @param {GC2Polynomial} p1 + * @param {GC2Polynomial} p2 + */ +export const addInto = (p1, p2) => { + p2.degrees.forEach(degree => { + if (p1.degrees.has(degree)) { + p1.degrees.delete(degree) + } else { + p1.degrees.add(degree) + } + }) +} + +/** + * Or (|) p2 into the p1 polynomial. + * + * Addition is defined as xor in F2. Substraction is equivalent to addition in F2. + * + * @param {GC2Polynomial} p1 + * @param {GC2Polynomial} p2 + */ +export const orInto = (p1, p2) => { + p2.degrees.forEach(degree => { + p1.degrees.add(degree) + }) +} + +/** + * Add (+) p2 to the p1 polynomial. + * + * Addition is defined as xor in F2. Substraction is equivalent to addition in F2. + * + * @param {GC2Polynomial} p1 + * @param {GC2Polynomial} p2 + */ +export const add = (p1, p2) => { + const result = new GC2Polynomial() + p2.degrees.forEach(degree => { + if (!p1.degrees.has(degree)) { + result.degrees.add(degree) + } + }) + p1.degrees.forEach(degree => { + if (!p2.degrees.has(degree)) { + result.degrees.add(degree) + } + }) + return result +} + +/** + * Add (+) p2 to the p1 polynomial. + * + * Addition is defined as xor in F2. Substraction is equivalent to addition in F2. + * + * @param {GC2Polynomial} p + */ +export const clone = (p) => { + const result = new GC2Polynomial() + p.degrees.forEach(d => result.degrees.add(d)) + return result +} + +/** + * Add (+) p2 to the p1 polynomial. + * + * Addition is defined as xor in F2. Substraction is equivalent to addition in F2. + * + * @param {GC2Polynomial} p + * @param {number} degree + */ +export const addDegreeInto = (p, degree) => { + if (p.degrees.has(degree)) { + p.degrees.delete(degree) + } else { + p.degrees.add(degree) + } +} + +/** + * Multiply (•) p1 with p2 and store the result in p1. + * + * @param {GC2Polynomial} p1 + * @param {GC2Polynomial} p2 + */ +export const multiply = (p1, p2) => { + const result = new GC2Polynomial() + p1.degrees.forEach(degree1 => { + p2.degrees.forEach(degree2 => { + addDegreeInto(result, degree1 + degree2) + }) + }) + return result +} + +/** + * Multiply (•) p1 with p2 and store the result in p1. + * + * @param {GC2Polynomial} p + * @param {number} shift + */ +export const shiftLeft = (p, shift) => { + const result = new GC2Polynomial() + p.degrees.forEach(degree => { + const r = degree + shift + r >= 0 && result.degrees.add(r) + }) + return result +} + +/** + * Multiply (•) p1 with p2 and store the result in p1. + * + * @param {GC2Polynomial} p + * @param {number} shift + */ +export const shiftRight = (p, shift) => shiftLeft(p, -shift) + +/** + * Computes p1 % p2. I.e. the remainder of p1/p2. + * + * @param {GC2Polynomial} p1 + * @param {GC2Polynomial} p2 + */ +export const mod = (p1, p2) => { + const maxDeg1 = getHighestDegree(p1) + const maxDeg2 = getHighestDegree(p2) + const result = clone(p1) + for (let i = maxDeg1 - maxDeg2; i >= 0; i--) { + if (result.degrees.has(maxDeg2 + i)) { + const shifted = shiftLeft(p2, i) + addInto(result, shifted) + } + } + return result +} + +/** + * Computes (p^e mod m). + * + * http://en.wikipedia.org/wiki/Modular_exponentiation + * + * @param {GC2Polynomial} p + * @param {number} e + * @param {GC2Polynomial} m + */ +export const modPow = (p, e, m) => { + let result = ONE + while (true) { + if ((e & 1) === 1) { + result = mod(multiply(result, p), m) + } + e >>>= 1 + if (e === 0) { + return result + } + p = mod(multiply(p, p), m) + } +} + +/** + * Find the greatest common divisor using Euclid's Algorithm. + * + * @param {GC2Polynomial} p1 + * @param {GC2Polynomial} p2 + */ +export const gcd = (p1, p2) => { + while (p2.degrees.size > 0) { + const modded = mod(p1, p2) + p1 = p2 + p2 = modded + } + return p1 +} + +/** + * true iff p1 equals p2 + * + * @param {GC2Polynomial} p1 + * @param {GC2Polynomial} p2 + */ +export const equals = (p1, p2) => { + if (p1.degrees.size !== p2.degrees.size) return false + for (const d of p1.degrees) { + if (!p2.degrees.has(d)) return false + } + return true +} + +const X = createFromBytes(new Uint8Array([2])) +const ONE = createFromBytes(new Uint8Array([1])) + +/** + * Computes ( x^(2^p) - x ) mod f + * + * (shamelessly copied from + * https://github.com/opendedup/rabinfingerprint/blob/master/src/org/rabinfingerprint/polynomial/Polynomial.java) + * + * @param {GC2Polynomial} f + * @param {number} p + */ +const reduceExponent = (f, p) => { + // compute (x^q^p mod f) + const q2p = math.pow(2, p) + const x2q2p = modPow(X, q2p, f) + // subtract (x mod f) + return mod(add(x2q2p, X), f) +} + +/** + * BenOr Reducibility Test + * + * Tests and Constructions of Irreducible Polynomials over Finite Fields + * (1997) Shuhong Gao, Daniel Panario + * + * http://citeseer.ist.psu.edu/cache/papers/cs/27167/http:zSzzSzwww.math.clemson.eduzSzfacultyzSzGaozSzpaperszSzGP97a.pdf/gao97tests.pdf + * + * @param {GC2Polynomial} p + */ +export const isIrreducibleBenOr = p => { + const degree = getHighestDegree(p) + for (let i = 1; i < degree / 2; i++) { + const b = reduceExponent(p, i) + const g = gcd(p, b) + if (!equals(g, ONE)) { + return false + } + } + return true +} + +/** + * @param {number} degree + */ +export const createIrreducible = degree => { + while (true) { + const p = createRandom(degree) + if (isIrreducibleBenOr(p)) return p + } +} + +/** + * Create a fingerprint of buf using the irreducible polynomial m. + * + * @param {Uint8Array} buf + * @param {GC2Polynomial} m + */ +export const fingerprint = (buf, m) => toUint8ArrayLsb(mod(createFromBytes(buf), m)) + +export class FingerprintEncoder { + /** + * @param {GC2Polynomial} m The irreducible polynomial + */ + constructor (m) { + this.fingerprint = new GC2Polynomial() + this.m = m + } + + /** + * @param {number} b + */ + write (b) { + const bp = createFromBytes(new Uint8Array([b])) + const fingerprint = shiftLeft(this.fingerprint, 8) + orInto(fingerprint, bp) + this.fingerprint = mod(fingerprint, this.m) + } + + getFingerprint () { + return toUint8ArrayLsb(this.fingerprint) + } +} diff --git a/crypto/gc2-polynomial.test.js b/crypto/gc2-polynomial.test.js new file mode 100644 index 0000000..d7a7ce7 --- /dev/null +++ b/crypto/gc2-polynomial.test.js @@ -0,0 +1,106 @@ +import * as t from '../testing.js' +import * as gc2 from './gc2-polynomial.js' +import * as math from '../math.js' +import * as array from '../array.js' +import * as prng from '../prng.js' +import * as buffer from '../buffer.js' + +/** + * @param {t.TestCase} _tc + */ +export const testPolynomialBasics = _tc => { + const bs = new Uint8Array([1, 11]) + const p = gc2.createFromBytes(bs) + t.assert(p.degrees.has(3)) + t.assert(p.degrees.has(1)) + t.assert(p.degrees.has(0)) + t.assert(p.degrees.has(8)) +} + +/** + * @param {t.TestCase} _tc + */ +export const testIrreducibleInput = _tc => { + const pa = gc2.createFromUint(0x53) + const pb = gc2.createFromUint(0xCA) + const pm = gc2.createFromUint(0x11B) + const px = gc2.multiply(pa, pb) + t.compare(new Uint8Array([0x53]), gc2.toUint8Array(pa)) + t.compare(new Uint8Array([0xCA]), gc2.toUint8Array(pb)) + t.assert(gc2.equals(gc2.createFromUint(0x3F7E), px)) + t.compare(new Uint8Array([0x3F, 0x7E]), gc2.toUint8Array(px)) + const pabm = gc2.mod(px, pm) + t.compare(new Uint8Array([0x1]), gc2.toUint8Array(pabm)) +} + +/** + * @param {t.TestCase} _tc + */ +export const testIrreducibleSpread = _tc => { + const degree = 53 + const N = 200 + const avgSpread = getSpreadAverage(degree, N) + const diffSpread = math.abs(avgSpread - degree) + t.info(`Average spread for degree ${degree} at ${N} repetitions: ${avgSpread}`) + t.assert(diffSpread < 3, 'Spread of irreducible polynomials is within expected range') +} + +/** + * @param {number} degree + * @param {number} tests + */ +const getSpreadAverage = (degree, tests) => { + const spreads = [] + for (let i = 0, test = 0, lastI = 0; test < tests; i++) { + const f = gc2.createRandom(degree) + t.assert(gc2.getHighestDegree(f) === degree) + if (gc2.isIrreducibleBenOr(f)) { + const spread = i - lastI + spreads.push(spread) + lastI = i + test++ + } + } + return array.fold(spreads, 0, math.add) / tests +} + +/** + * @param {t.TestCase} tc + */ +export const testFingerprint = tc => { + /** + * @type {Array} + */ + const dataObjects = [] + const N = 3000 + const K = 53 + const MSIZE = 130 + const irreducible = gc2.createIrreducible(K) + t.info(`N=${N} K=${K} MSIZE=${MSIZE}`) + for (let i = 0; i < N; i++) { + dataObjects.push(prng.uint8Array(tc.prng, MSIZE)) + } + /** + * @type {Array} + */ + let fingerprints1 = [] + t.measureTime('polynomial direct', () => { + fingerprints1 = dataObjects.map((o, _index) => gc2.fingerprint(o, irreducible)) + }) + const testSet = new Set(fingerprints1.map(buffer.toBase64)) + t.assert(testSet.size === N) + /** + * @type {Array} + */ + let fingerprints2 = [] + t.measureTime('polynomial incremental', () => { + fingerprints2 = dataObjects.map((o, _index) => { + const encoder = new gc2.FingerprintEncoder(irreducible) + for (let i = 0; i < o.byteLength; i++) { + encoder.write(o[i]) + } + return encoder.getFingerprint() + }) + }) + t.compare(fingerprints1, fingerprints2) +} diff --git a/number.js b/number.js index 575967e..bbc297a 100644 --- a/number.js +++ b/number.js @@ -18,3 +18,20 @@ export const HIGHEST_UINT32 = binary.BITS32 export const isInteger = Number.isInteger || (num => typeof num === 'number' && isFinite(num) && math.floor(num) === num) export const isNaN = Number.isNaN export const parseInt = Number.parseInt + +/** + * Count the number of "1" bits in an unsigned 32bit number. + * + * Super fun bitcount algorithm by Brian Kernighan. + * + * @param {number} n + */ +export const countBits = n => { + n &= binary.BITS32 + let count = 0 + while (n) { + n &= (n - 1) + count++ + } + return count +} diff --git a/test.js b/test.js index bd32d3c..009589f 100644 --- a/test.js +++ b/test.js @@ -2,6 +2,7 @@ import { runTests } from './testing.js' import * as array from './array.test.js' import * as broadcastchannel from './broadcastchannel.test.js' import * as crypto from './crypto.test.js' +import * as cryptoPolynomial from './crypto/gc2-polynomial.test.js' import * as logging from './logging.test.js' import * as string from './string.test.js' import * as encoding from './encoding.test.js' @@ -44,6 +45,7 @@ runTests({ array, broadcastchannel, crypto, + cryptoPolynomial, logging, string, encoding,