Benchmark: Inline code from benchmark.js

package.json

@@ -49,7 +49,6 @@
     "@babel/preset-env": "7.4.5",
     "@babel/register": "7.4.4",
     "babel-eslint": "10.0.2",
-    "benchmark": "2.1.4",
     "chai": "4.2.0",
     "eslint": "5.16.0",
     "eslint-plugin-flowtype": "3.11.1",

resources/benchmark.js

@@ -5,7 +5,6 @@
 const os = require('os');
 const fs = require('fs');
 const path = require('path');
-const { Benchmark } = require('benchmark');
 
 const {
   exec,
@@ -16,6 +15,7 @@ const {
   readdirRecursive,
 } = require('./utils');
 
+const NS_PER_SEC = 1e9;
 const LOCAL = 'local';
 
 function LOCAL_DIR(...paths) {
@@ -93,12 +93,20 @@ function runBenchmark(benchmark, environments) {
   const benches = environments.map(environment => {
     const module = require(path.join(environment.distPath, benchmark));
     benchmarkName = module.name;
-    return new Benchmark(environment.revision, module.measure);
+    return {
+      name: environment.revision,
+      fn: module.measure,
+    };
   });
 
   console.log('⏱️   ' + benchmarkName);
   for (let i = 0; i < benches.length; ++i) {
-    benches[i].run({ async: false });
+    const bench = benches[i];
+    try {
+      bench.sample = run(bench.fn);
+    } catch (e) {
+      bench.error = e;
+    }
     process.stdout.write('  ' + cyan(i + 1) + ' tests completed.\u000D');
   }
   console.log('\n');
@@ -107,14 +115,123 @@ function runBenchmark(benchmark, environments) {
   console.log('');
 }
 
+// Clocks the time taken to execute a test per cycle (secs).
+function clock(count, fn) {
+  const start = process.hrtime.bigint();
+  for (let i = count; count !== 0; --count) {
+    fn();
+  }
+  return Number(process.hrtime.bigint() - start);
+}
+
+// Cycles a benchmark until a run `count` can be established.
+function cycle(initCount, fn) {
+  // Resolve time span required to achieve a percent uncertainty of at most 1%.
+  // For more information see http://spiff.rit.edu/classes/phys273/uncert/uncert.html.
+  const minTime = 0.05 * NS_PER_SEC;
+  let count = initCount;
+  let clocked = 0;
+
+  while (true) {
+    clocked = clock(count, fn);
+
+    // Do we need to do another cycle?
+    if (clocked >= minTime) {
+      break;
+    }
+
+    // Calculate how many more iterations it will take to achieve the `minTime`.
+    count += Math.ceil((minTime - clocked) / clocked * count);
+  }
+
+  return count;
+}
+
+// Computes stats on benchmark results.
+function run(fn) {
+  const bench = this;
+  const sample = [];
+
+  // The maximum time a benchmark is allowed to run before finishing.
+  const maxTime = 5 * NS_PER_SEC;
+  // The minimum sample size required to perform statistical analysis.
+  const minSamples = 15;
+  // The default number of times to execute a test on a benchmark's first cycle.
+  const initCount = 10;
+
+  const count = cycle(initCount, fn);
+  clock(count, fn); // initial warm up
+  // If time permits, increase sample size to reduce the margin of error.
+  let elapsed = 0;
+  while (sample.length < minSamples || elapsed < maxTime) {
+    const clocked = clock(count, fn);
+    elapsed += clocked;
+    // Compute the seconds per operation.
+    sample.push(clocked / count);
+  }
+
+  return sample;
+}
+
+/*
+ * T-Distribution two-tailed critical values for 95% confidence.
+ * For more info see http://www.itl.nist.gov/div898/handbook/eda/section3/eda3672.htm.
+ */
+var tTable = {
+  '1':  12.706, '2':  4.303, '3':  3.182, '4':  2.776, '5':  2.571, '6':  2.447,
+  '7':  2.365,  '8':  2.306, '9':  2.262, '10': 2.228, '11': 2.201, '12': 2.179,
+  '13': 2.16,   '14': 2.145, '15': 2.131, '16': 2.12,  '17': 2.11,  '18': 2.101,
+  '19': 2.093,  '20': 2.086, '21': 2.08,  '22': 2.074, '23': 2.069, '24': 2.064,
+  '25': 2.06,   '26': 2.056, '27': 2.052, '28': 2.048, '29': 2.045, '30': 2.042,
+  'infinity': 1.96
+};
+
 function beautifyBenchmark(results) {
-  const benches = results.map(result => ({
-    name: result.name,
-    error: result.error,
-    ops: result.hz,
-    deviation: result.stats.rme,
-    numRuns: result.stats.sample.length,
-  }));
+  const benches = results.map(result => {
+    const sample = result.sample || [];
+
+    // Compute the sample mean (estimate of the population mean).
+    let mean = 0;
+    for (const x of sample) {
+      mean += x;
+    }
+    mean /= sample.length;
+    mean = mean || 0;
+
+    // Compute the sample variance (estimate of the population variance).
+    let variance = 0;
+    for (const x of sample) {
+      variance += Math.pow(x - mean, 2);
+    }
+    variance /= (sample.length - 1);
+    variance = variance || 0;
+
+    // Compute the sample standard deviation (estimate of the population standard deviation).
+    const sd = Math.sqrt(variance);
+
+    // Compute the standard error of the mean (a.k.a. the standard deviation of the sampling distribution of the sample mean).
+    const sem = sd / Math.sqrt(sample.length);
+
+    // Compute the degrees of freedom.
+    const df = sample.length - 1;
+
+    // Compute the critical value.
+    const critical = tTable[Math.round(df) || 1] || tTable.infinity;
+
+    // Compute the margin of error.
+    const moe = sem * critical;
+
+    // The relative margin of error (expressed as a percentage of the mean).
+    const rme = (moe / mean) * 100 || 0;
+
+    return {
+      name: result.name,
+      error: result.error,
+      ops: NS_PER_SEC / mean,
+      deviation: rme,
+      numRuns: sample.length,
+    };
+  });
 
   const nameMaxLen = maxBy(benches, ({ name }) => name.length);
   const opsTop = maxBy(benches, ({ ops }) => ops);

yarn.lock

@@ -751,14 +751,6 @@ balanced-match@^1.0.0:
   resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
   integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c=
 
-benchmark@2.1.4:
-  version "2.1.4"
-  resolved "https://registry.yarnpkg.com/benchmark/-/benchmark-2.1.4.tgz#09f3de31c916425d498cc2ee565a0ebf3c2a5629"
-  integrity sha1-CfPeMckWQl1JjMLuVloOvzwqVik=
-  dependencies:
-    lodash "^4.17.4"
-    platform "^1.3.3"
-
 brace-expansion@^1.1.7:
   version "1.1.11"
   resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd"
@@ -1699,7 +1691,7 @@ lodash.flattendeep@^4.4.0:
   resolved "https://registry.yarnpkg.com/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz#fb030917f86a3134e5bc9bec0d69e0013ddfedb2"
   integrity sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=
 
-lodash@^4.17.11, lodash@^4.17.4:
+lodash@^4.17.11:
   version "4.17.11"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d"
   integrity sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==
@@ -2129,11 +2121,6 @@ pkg-dir@^3.0.0:
   dependencies:
     find-up "^3.0.0"
 
-platform@^1.3.3:
-  version "1.3.5"
-  resolved "https://registry.yarnpkg.com/platform/-/platform-1.3.5.tgz#fb6958c696e07e2918d2eeda0f0bc9448d733444"
-  integrity sha512-TuvHS8AOIZNAlE77WUDiR4rySV/VMptyMfcfeoMgs4P8apaZM3JrnbzBiixKUv+XR6i+BXrQh8WAnjaSPFO65Q==
-
 prelude-ls@~1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54"