diff --git a/package.json b/package.json index 8f1dedcf45..6ffaa3a073 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,6 @@ "@babel/preset-env": "7.4.5", "@babel/register": "7.4.4", "babel-eslint": "10.0.2", - "benchmark": "2.1.4", "chai": "4.2.0", "eslint": "5.16.0", "eslint-plugin-flowtype": "3.11.1", diff --git a/resources/benchmark.js b/resources/benchmark.js index 247536fc61..f77f7f4c64 100644 --- a/resources/benchmark.js +++ b/resources/benchmark.js @@ -5,8 +5,9 @@ const os = require('os'); const fs = require('fs'); const path = require('path'); -const { Benchmark } = require('benchmark'); +const assert = require('assert'); +const { red, green, yellow, cyan, grey } = require('./colors'); const { exec, copyFile, @@ -16,8 +17,17 @@ const { readdirRecursive, } = require('./utils'); +const NS_PER_SEC = 1e9; const LOCAL = 'local'; +const minTime = 0.05 * NS_PER_SEC; +// The maximum time a benchmark is allowed to run before finishing. +const maxTime = 5 * NS_PER_SEC; +// The minimum sample size required to perform statistical analysis. +const minSamples = 15; +// The default number of times to execute a test on a benchmark's first cycle. +const initCount = 10; + function LOCAL_DIR(...paths) { return path.join(__dirname, '..', ...paths); } @@ -93,43 +103,132 @@ function runBenchmark(benchmark, environments) { const benches = environments.map(environment => { const module = require(path.join(environment.distPath, benchmark)); benchmarkName = module.name; - return new Benchmark(environment.revision, module.measure); + return { + name: environment.revision, + fn: module.measure, + }; }); console.log('⏱️ ' + benchmarkName); + const results = []; for (let i = 0; i < benches.length; ++i) { - benches[i].run({ async: false }); - process.stdout.write(' ' + cyan(i + 1) + ' tests completed.\u000D'); + const { name, fn } = benches[i]; + try { + const samples = collectSamples(fn); + results.push({ name, samples, ...computeStats(samples) }); + process.stdout.write(' ' + cyan(i + 1) + ' tests completed.\u000D'); + } catch (error) { + console.log(' ' + name + ': ' + red(String(error))); + } } console.log('\n'); - beautifyBenchmark(benches); + beautifyBenchmark(results); console.log(''); } -function beautifyBenchmark(results) { - const benches = results.map(result => ({ - name: result.name, - error: result.error, - ops: result.hz, - deviation: result.stats.rme, - numRuns: result.stats.sample.length, - })); +function collectSamples(fn) { + clock(initCount, fn); // initial warm up + + // Cycles a benchmark until a run `count` can be established. + // Resolve time span required to achieve a percent uncertainty of at most 1%. + // For more information see http://spiff.rit.edu/classes/phys273/uncert/uncert.html. + let count = initCount; + let clocked = 0; + while ((clocked = clock(count, fn)) < minTime) { + // Calculate how many more iterations it will take to achieve the `minTime`. + count += Math.ceil(((minTime - clocked) * count) / clocked); + } - const nameMaxLen = maxBy(benches, ({ name }) => name.length); - const opsTop = maxBy(benches, ({ ops }) => ops); - const opsMaxLen = maxBy(benches, ({ ops }) => beautifyNumber(ops).length); + let elapsed = 0; + const samples = []; - for (const bench of benches) { - if (bench.error) { - console.log(' ' + bench.name + ': ' + red(String(bench.error))); - continue; - } - printBench(bench); + // If time permits, increase sample size to reduce the margin of error. + while (samples.length < minSamples || elapsed < maxTime) { + clocked = clock(count, fn); + assert(clocked > 0); + + elapsed += clocked; + // Compute the seconds per operation. + samples.push(clocked / count); + } + + return samples; +} + +// Clocks the time taken to execute a test per cycle (secs). +function clock(count, fn) { + const start = process.hrtime.bigint(); + for (let i = 0; i < count; ++i) { + fn(); + } + return Number(process.hrtime.bigint() - start); +} + +// T-Distribution two-tailed critical values for 95% confidence. +// See http://www.itl.nist.gov/div898/handbook/eda/section3/eda3672.htm. +const tTable = /* prettier-ignore */ { + '1': 12.706, '2': 4.303, '3': 3.182, '4': 2.776, '5': 2.571, '6': 2.447, + '7': 2.365, '8': 2.306, '9': 2.262, '10': 2.228, '11': 2.201, '12': 2.179, + '13': 2.16, '14': 2.145, '15': 2.131, '16': 2.12, '17': 2.11, '18': 2.101, + '19': 2.093, '20': 2.086, '21': 2.08, '22': 2.074, '23': 2.069, '24': 2.064, + '25': 2.06, '26': 2.056, '27': 2.052, '28': 2.048, '29': 2.045, '30': 2.042, + infinity: 1.96, +}; + +// Computes stats on benchmark results. +function computeStats(samples) { + assert(samples.length > 1); + + // Compute the sample mean (estimate of the population mean). + let mean = 0; + for (const x of samples) { + mean += x; + } + mean /= samples.length; + + // Compute the sample variance (estimate of the population variance). + let variance = 0; + for (const x of samples) { + variance += Math.pow(x - mean, 2); + } + variance /= samples.length - 1; + + // Compute the sample standard deviation (estimate of the population standard deviation). + const sd = Math.sqrt(variance); + + // Compute the standard error of the mean (a.k.a. the standard deviation of the sampling distribution of the sample mean). + const sem = sd / Math.sqrt(samples.length); + + // Compute the degrees of freedom. + const df = samples.length - 1; + + // Compute the critical value. + const critical = tTable[df] || tTable.infinity; + + // Compute the margin of error. + const moe = sem * critical; + + // The relative margin of error (expressed as a percentage of the mean). + const rme = (moe / mean) * 100 || 0; + + return { + ops: NS_PER_SEC / mean, + deviation: rme, + }; +} + +function beautifyBenchmark(results) { + const nameMaxLen = maxBy(results, ({ name }) => name.length); + const opsTop = maxBy(results, ({ ops }) => ops); + const opsMaxLen = maxBy(results, ({ ops }) => beautifyNumber(ops).length); + + for (const result of results) { + printBench(result); } function printBench(bench) { - const { name, ops, deviation, numRuns } = bench; + const { name, ops, deviation, samples } = bench; console.log( ' ' + nameStr() + @@ -139,7 +238,7 @@ function beautifyBenchmark(results) { grey('\xb1') + deviationStr() + cyan('%') + - grey(' (' + numRuns + ' runs sampled)'), + grey(' (' + samples.length + ' runs sampled)'), ); function nameStr() { @@ -160,22 +259,6 @@ function beautifyBenchmark(results) { } } -function red(str) { - return '\u001b[31m' + str + '\u001b[0m'; -} -function green(str) { - return '\u001b[32m' + str + '\u001b[0m'; -} -function yellow(str) { - return '\u001b[33m' + str + '\u001b[0m'; -} -function cyan(str) { - return '\u001b[36m' + str + '\u001b[0m'; -} -function grey(str) { - return '\u001b[90m' + str + '\u001b[0m'; -} - function beautifyNumber(num) { return Number(num.toFixed(num > 100 ? 0 : 2)).toLocaleString(); } diff --git a/resources/colors.js b/resources/colors.js new file mode 100644 index 0000000000..a9ce45a698 --- /dev/null +++ b/resources/colors.js @@ -0,0 +1,31 @@ +// @noflow + +'use strict'; + +function red(str) { + return '\u001b[31m' + str + '\u001b[0m'; +} + +function green(str) { + return '\u001b[32m' + str + '\u001b[0m'; +} + +function yellow(str) { + return '\u001b[33m' + str + '\u001b[0m'; +} + +function cyan(str) { + return '\u001b[36m' + str + '\u001b[0m'; +} + +function grey(str) { + return '\u001b[90m' + str + '\u001b[0m'; +} + +module.exports = { + red, + green, + yellow, + cyan, + grey, +}; diff --git a/yarn.lock b/yarn.lock index b5de0d9732..44f39cb1b0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -751,14 +751,6 @@ balanced-match@^1.0.0: resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c= -benchmark@2.1.4: - version "2.1.4" - resolved "https://registry.yarnpkg.com/benchmark/-/benchmark-2.1.4.tgz#09f3de31c916425d498cc2ee565a0ebf3c2a5629" - integrity sha1-CfPeMckWQl1JjMLuVloOvzwqVik= - dependencies: - lodash "^4.17.4" - platform "^1.3.3" - brace-expansion@^1.1.7: version "1.1.11" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" @@ -1699,7 +1691,7 @@ lodash.flattendeep@^4.4.0: resolved "https://registry.yarnpkg.com/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz#fb030917f86a3134e5bc9bec0d69e0013ddfedb2" integrity sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI= -lodash@^4.17.11, lodash@^4.17.4: +lodash@^4.17.11: version "4.17.11" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d" integrity sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg== @@ -2129,11 +2121,6 @@ pkg-dir@^3.0.0: dependencies: find-up "^3.0.0" -platform@^1.3.3: - version "1.3.5" - resolved "https://registry.yarnpkg.com/platform/-/platform-1.3.5.tgz#fb6958c696e07e2918d2eeda0f0bc9448d733444" - integrity sha512-TuvHS8AOIZNAlE77WUDiR4rySV/VMptyMfcfeoMgs4P8apaZM3JrnbzBiixKUv+XR6i+BXrQh8WAnjaSPFO65Q== - prelude-ls@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54"