benchmark: add test and all options and improve errors"

This reverts commit 4671d55 and
contains a fix to the issue raised for the revert.

PR-URL: #31755
Reviewed-By: Anna Henningsen <anna@addaleax.net>
Reviewed-By: Vladimir de Turckheim <vlad2t@hotmail.com>
Reviewed-By: Rich Trott <rtrott@gmail.com>

Author: BridgeAR

benchmark/_cli.js

@@ -6,22 +6,24 @@ const path = require('path');
 // Create an object of all benchmark scripts
 const benchmarks = {};
 fs.readdirSync(__dirname)
-  .filter((name) => fs.statSync(path.resolve(__dirname, name)).isDirectory())
+  .filter((name) => {
+    return name !== 'fixtures' &&
+           fs.statSync(path.resolve(__dirname, name)).isDirectory();
+  })
   .forEach((category) => {
     benchmarks[category] = fs.readdirSync(path.resolve(__dirname, category))
       .filter((filename) => filename[0] !== '.' && filename[0] !== '_');
   });
 
 function CLI(usage, settings) {
-  if (!(this instanceof CLI)) return new CLI(usage, settings);
-
   if (process.argv.length < 3) {
     this.abort(usage); // Abort will exit the process
   }
 
   this.usage = usage;
   this.optional = {};
   this.items = [];
+  this.test = false;
 
   for (const argName of settings.arrayArgs) {
     this.optional[argName] = [];
@@ -34,7 +36,7 @@ function CLI(usage, settings) {
     if (arg === '--') {
       // Only items can follow --
       mode = 'item';
-    } else if ('both' === mode && arg[0] === '-') {
+    } else if (mode === 'both' && arg[0] === '-') {
       // Optional arguments declaration
 
       if (arg[1] === '-') {
@@ -61,6 +63,8 @@ function CLI(usage, settings) {
 
       // The next value can be either an option or an item
       mode = 'both';
+    } else if (arg === 'test') {
+      this.test = true;
     } else if (['both', 'item'].includes(mode)) {
       // item arguments
       this.items.push(arg);
@@ -83,9 +87,15 @@ CLI.prototype.abort = function(msg) {
 CLI.prototype.benchmarks = function() {
   const paths = [];
 
+  if (this.items.includes('all')) {
+    this.items = Object.keys(benchmarks);
+  }
+
   for (const category of this.items) {
-    if (benchmarks[category] === undefined)
-      continue;
+    if (benchmarks[category] === undefined) {
+      console.error(`The "${category}" category does not exist.`);
+      process.exit(1);
+    }
     for (const scripts of benchmarks[category]) {
       if (this.shouldSkip(scripts)) continue;
 

benchmark/_http-benchmarkers.js

@@ -43,9 +43,8 @@ class AutocannonBenchmarker {
     }
     if (!result || !result.requests || !result.requests.average) {
       return undefined;
-    } else {
-      return result.requests.average;
     }
+    return result.requests.average;
   }
 }
 
@@ -58,10 +57,13 @@ class WrkBenchmarker {
   }
 
   create(options) {
+    const duration = typeof options.duration === 'number' ?
+      Math.max(options.duration, 1) :
+      options.duration;
     const args = [
-      '-d', options.duration,
+      '-d', duration,
       '-c', options.connections,
-      '-t', 8,
+      '-t', Math.min(options.connections, require('os').cpus().length || 8),
       `http://127.0.0.1:${options.port}${options.path}`,
     ];
     for (const field in options.headers) {
@@ -77,9 +79,8 @@ class WrkBenchmarker {
     const throughput = match && +match[1];
     if (!isFinite(throughput)) {
       return undefined;
-    } else {
-      return throughput;
     }
+    return throughput;
   }
 }
 
@@ -89,18 +90,21 @@ class WrkBenchmarker {
  */
 class TestDoubleBenchmarker {
   constructor(type) {
-    // `type` is the type ofbenchmarker. Possible values are 'http' and 'http2'.
+    // `type` is the type of benchmarker. Possible values are 'http' and
+    // 'http2'.
     this.name = `test-double-${type}`;
     this.executable = path.resolve(__dirname, '_test-double-benchmarker.js');
     this.present = fs.existsSync(this.executable);
     this.type = type;
   }
 
   create(options) {
-    const env = Object.assign({
-      duration: options.duration,
+    process.env.duration = process.env.duration || options.duration || 5;
+
+    const env = {
       test_url: `http://127.0.0.1:${options.port}${options.path}`,
-    }, process.env);
+      ...process.env
+    };
 
     const child = child_process.fork(this.executable,
                                      [this.type],
@@ -189,13 +193,14 @@ http_benchmarkers.forEach((benchmarker) => {
 });
 
 exports.run = function(options, callback) {
-  options = Object.assign({
+  options = {
     port: exports.PORT,
     path: '/',
     connections: 100,
     duration: 5,
     benchmarker: exports.default_http_benchmarker,
-  }, options);
+    ...options
+  };
   if (!options.benchmarker) {
     callback(new Error('Could not locate required http benchmarker. See ' +
                        `${requirementsURL} for further instructions.`));
@@ -220,7 +225,8 @@ exports.run = function(options, callback) {
   child.stderr.pipe(process.stderr);
 
   let stdout = '';
-  child.stdout.on('data', (chunk) => stdout += chunk.toString());
+  child.stdout.setEncoding('utf8');
+  child.stdout.on('data', (chunk) => stdout += chunk);
 
   child.once('close', (code) => {
     const elapsed = process.hrtime(benchmarker_start);

benchmark/_test-double-benchmarker.js

@@ -7,7 +7,7 @@ if (!['http', 'http2'].includes(myModule)) {
 
 const http = require(myModule);
 
-const duration = process.env.duration || 0;
+const duration = +process.env.duration;
 const url = process.env.test_url;
 
 const start = process.hrtime();
@@ -18,13 +18,15 @@ function request(res, client) {
   res.on('error', () => {});
   res.on('end', () => {
     throughput++;
-    const diff = process.hrtime(start);
-    if (duration > 0 && diff[0] < duration) {
+    const [sec, nanosec] = process.hrtime(start);
+    const ms = sec * 1000 + nanosec / 1e6;
+    if (ms < duration * 1000) {
       run();
     } else {
       console.log(JSON.stringify({ throughput }));
       if (client) {
         client.destroy();
+        process.exit(0);
       }
     }
   });
@@ -33,7 +35,7 @@ function request(res, client) {
 function run() {
   if (http.get) { // HTTP
     http.get(url, request);
-  } else {        // HTTP/2
+  } else { // HTTP/2
     const client = http.connect(url);
     client.on('error', (e) => { throw e; });
     request(client.request(), client);

benchmark/async_hooks/async-resource-vs-destroy.js

@@ -13,14 +13,12 @@ const {
 } = require('async_hooks');
 const { createServer } = require('http');
 
-// Configuration for the http server
-// there is no need for parameters in this test
-const connections = 500;
-const path = '/';
-
 const bench = common.createBenchmark(main, {
   type: ['async-resource', 'destroy', 'async-local-storage'],
   asyncMethod: ['callbacks', 'async'],
+  path: '/',
+  connections: 500,
+  duration: 5,
   n: [1e6]
 });
 
@@ -165,7 +163,7 @@ const asyncMethods = {
   'async': getServeAwait
 };
 
-function main({ type, asyncMethod }) {
+function main({ type, asyncMethod, connections, duration, path }) {
   const { server, close } = types[type](asyncMethods[asyncMethod]);
 
   server
@@ -174,7 +172,8 @@ function main({ type, asyncMethod }) {
 
       bench.http({
         path,
-        connections
+        connections,
+        duration
       }, () => {
         close();
       });

benchmark/async_hooks/http-server.js

@@ -3,10 +3,11 @@ const common = require('../common.js');
 
 const bench = common.createBenchmark(main, {
   asyncHooks: ['init', 'before', 'after', 'all', 'disabled', 'none'],
-  connections: [50, 500]
+  connections: [50, 500],
+  duration: 5
 });
 
-function main({ asyncHooks, connections }) {
+function main({ asyncHooks, connections, duration }) {
   if (asyncHooks !== 'none') {
     let hooks = {
       init() {},
@@ -33,6 +34,7 @@ function main({ asyncHooks, connections }) {
       bench.http({
         connections,
         path,
+        duration
       }, () => {
         server.close();
       });

benchmark/buffers/buffer-base64-encode.js

@@ -25,6 +25,8 @@ const common = require('../common.js');
 const bench = common.createBenchmark(main, {
   len: [64 * 1024 * 1024],
   n: [32]
+}, {
+  test: { len: 256 }
 });
 
 function main({ n, len }) {

benchmark/buffers/buffer-swap.js

@@ -7,6 +7,8 @@ const bench = common.createBenchmark(main, {
   method: ['swap16', 'swap32', 'swap64'/* , 'htons', 'htonl', 'htonll' */],
   len: [64, 256, 768, 1024, 2056, 8192],
   n: [1e6]
+}, {
+  test: { len: 16 }
 });
 
 // The htons and htonl methods below are used to benchmark the

benchmark/common.js

@@ -4,43 +4,83 @@ const child_process = require('child_process');
 const http_benchmarkers = require('./_http-benchmarkers.js');
 
 class Benchmark {
-  constructor(fn, configs, options) {
-    // Use the file name as the name of the benchmark
-    this.name = require.main.filename.slice(__dirname.length + 1);
+  // Used to make sure a benchmark only start a timer once
+  #started = false;
+
+  // Indicate that the benchmark ended
+  #ended = false;
+
+  // Holds process.hrtime value
+  #time = [0, 0];
+
+  // Use the file name as the name of the benchmark
+  name = require.main.filename.slice(__dirname.length + 1);
+
+  // Execution arguments i.e. flags used to run the jobs
+  flags = process.env.NODE_BENCHMARK_FLAGS ?
+    process.env.NODE_BENCHMARK_FLAGS.split(/\s+/) :
+    [];
+
+  constructor(fn, configs, options = {}) {
     // Parse job-specific configuration from the command line arguments
-    const parsed_args = this._parseArgs(process.argv.slice(2), configs);
+    const argv = process.argv.slice(2);
+    const parsed_args = this._parseArgs(argv, configs, options);
     this.options = parsed_args.cli;
     this.extra_options = parsed_args.extra;
+    if (options.flags) {
+      this.flags = this.flags.concat(options.flags);
+    }
+
     // The configuration list as a queue of jobs
     this.queue = this._queue(this.options);
+
     // The configuration of the current job, head of the queue
     this.config = this.queue[0];
-    // Execution arguments i.e. flags used to run the jobs
-    this.flags = [];
-    if (options && options.flags) {
-      this.flags = this.flags.concat(options.flags);
-    }
-    if (process.env.NODE_BENCHMARK_FLAGS) {
-      const flags = process.env.NODE_BENCHMARK_FLAGS.split(/\s+/);
-      this.flags = this.flags.concat(flags);
-    }
-    // Holds process.hrtime value
-    this._time = [0, 0];
-    // Used to make sure a benchmark only start a timer once
-    this._started = false;
-    this._ended = false;
-
-    // this._run will use fork() to create a new process for each configuration
-    // combination.
-    if (process.env.hasOwnProperty('NODE_RUN_BENCHMARK_FN')) {
-      process.nextTick(() => fn(this.config));
-    } else {
-      process.nextTick(() => this._run());
-    }
+
+    process.nextTick(() => {
+      if (process.env.hasOwnProperty('NODE_RUN_BENCHMARK_FN')) {
+        fn(this.config);
+      } else {
+        // _run will use fork() to create a new process for each configuration
+        // combination.
+        this._run();
+      }
+    });
   }
 
-  _parseArgs(argv, configs) {
+  _parseArgs(argv, configs, options) {
     const cliOptions = {};
+
+    // Check for the test mode first.
+    const testIndex = argv.indexOf('--test');
+    if (testIndex !== -1) {
+      for (const [key, rawValue] of Object.entries(configs)) {
+        let value = Array.isArray(rawValue) ? rawValue[0] : rawValue;
+        // Set numbers to one by default to reduce the runtime.
+        if (typeof value === 'number') {
+          if (key === 'dur' || key === 'duration') {
+            value = 0.05;
+          } else if (value > 1) {
+            value = 1;
+          }
+        }
+        cliOptions[key] = [value];
+      }
+      // Override specific test options.
+      if (options.test) {
+        for (const [key, value] of Object.entries(options.test)) {
+          cliOptions[key] = Array.isArray(value) ? value : [value];
+        }
+      }
+      argv.splice(testIndex, 1);
+    } else {
+      // Accept single values instead of arrays.
+      for (const [key, value] of Object.entries(configs)) {
+        if (!Array.isArray(value))
+          configs[key] = [value];
+      }
+    }
+
     const extraOptions = {};
     const validArgRE = /^(.+?)=([\s\S]*)$/;
     // Parse configuration arguments
@@ -50,45 +90,43 @@ class Benchmark {
         console.error(`bad argument: ${arg}`);
         process.exit(1);
       }
-      const config = match[1];
-
-      if (configs[config]) {
-        // Infer the type from the config object and parse accordingly
-        const isNumber = typeof configs[config][0] === 'number';
-        const value = isNumber ? +match[2] : match[2];
-        if (!cliOptions[config])
-          cliOptions[config] = [];
-        cliOptions[config].push(value);
+      const [, key, value] = match;
+      if (Object.prototype.hasOwnProperty.call(configs, key)) {
+        if (!cliOptions[key])
+          cliOptions[key] = [];
+        cliOptions[key].push(
+          // Infer the type from the config object and parse accordingly
+          typeof configs[key][0] === 'number' ? +value : value
+        );
       } else {
-        extraOptions[config] = match[2];
+        extraOptions[key] = value;
       }
     }
-    return { cli: Object.assign({}, configs, cliOptions), extra: extraOptions };
+    return { cli: { ...configs, ...cliOptions }, extra: extraOptions };
   }
 
   _queue(options) {
     const queue = [];
     const keys = Object.keys(options);
 
-    // Perform a depth-first walk though all options to generate a
+    // Perform a depth-first walk through all options to generate a
     // configuration list that contains all combinations.
     function recursive(keyIndex, prevConfig) {
       const key = keys[keyIndex];
       const values = options[key];
-      const type = typeof values[0];
 
       for (const value of values) {
         if (typeof value !== 'number' && typeof value !== 'string') {
           throw new TypeError(
             `configuration "${key}" had type ${typeof value}`);
         }
-        if (typeof value !== type) {
+        if (typeof value !== typeof values[0]) {
           // This is a requirement for being able to consistently and
           // predictably parse CLI provided configuration values.
           throw new TypeError(`configuration "${key}" has mixed types`);
         }
 
-        const currConfig = Object.assign({ [key]: value }, prevConfig);
+        const currConfig = { [key]: value, ...prevConfig };
 
         if (keyIndex + 1 < keys.length) {
           recursive(keyIndex + 1, currConfig);
@@ -108,12 +146,11 @@ class Benchmark {
   }
 
   http(options, cb) {
-    const self = this;
-    const http_options = Object.assign({ }, options);
+    const http_options = { ...options };
     http_options.benchmarker = http_options.benchmarker ||
-                              self.config.benchmarker ||
-                              self.extra_options.benchmarker ||
-                              exports.default_http_benchmarker;
+                               this.config.benchmarker ||
+                               this.extra_options.benchmarker ||
+                               http_benchmarkers.default_http_benchmarker;
     http_benchmarkers.run(
       http_options, (error, code, used_benchmarker, result, elapsed) => {
         if (cb) {
@@ -123,14 +160,13 @@ class Benchmark {
           console.error(error);
           process.exit(code || 1);
         }
-        self.config.benchmarker = used_benchmarker;
-        self.report(result, elapsed);
+        this.config.benchmarker = used_benchmarker;
+        this.report(result, elapsed);
       }
     );
   }
 
   _run() {
-    const self = this;
     // If forked, report to the parent.
     if (process.send) {
       process.send({
@@ -140,57 +176,59 @@ class Benchmark {
       });
     }
 
-    (function recursive(queueIndex) {
-      const config = self.queue[queueIndex];
+    const recursive = (queueIndex) => {
+      const config = this.queue[queueIndex];
 
       // Set NODE_RUN_BENCHMARK_FN to indicate that the child shouldn't
       // construct a configuration queue, but just execute the benchmark
       // function.
-      const childEnv = Object.assign({}, process.env);
+      const childEnv = { ...process.env };
       childEnv.NODE_RUN_BENCHMARK_FN = '';
 
       // Create configuration arguments
       const childArgs = [];
-      for (const key of Object.keys(config)) {
-        childArgs.push(`${key}=${config[key]}`);
+      for (const [key, value] of Object.entries(config)) {
+        childArgs.push(`${key}=${value}`);
       }
-      for (const key of Object.keys(self.extra_options)) {
-        childArgs.push(`${key}=${self.extra_options[key]}`);
+      for (const [key, value] of Object.entries(this.extra_options)) {
+        childArgs.push(`${key}=${value}`);
       }
 
       const child = child_process.fork(require.main.filename, childArgs, {
         env: childEnv,
-        execArgv: self.flags.concat(process.execArgv),
+        execArgv: this.flags.concat(process.execArgv),
       });
       child.on('message', sendResult);
       child.on('close', (code) => {
         if (code) {
           process.exit(code);
         }
 
-        if (queueIndex + 1 < self.queue.length) {
+        if (queueIndex + 1 < this.queue.length) {
           recursive(queueIndex + 1);
         }
       });
-    })(0);
+    };
+
+    recursive(0);
   }
 
   start() {
-    if (this._started) {
+    if (this.#started) {
       throw new Error('Called start more than once in a single benchmark');
     }
-    this._started = true;
-    this._time = process.hrtime();
+    this.#started = true;
+    this.#time = process.hrtime();
   }
 
   end(operations) {
     // Get elapsed time now and do error checking later for accuracy.
-    const elapsed = process.hrtime(this._time);
+    const elapsed = process.hrtime(this.#time);
 
-    if (!this._started) {
+    if (!this.#started) {
       throw new Error('called end without start');
     }
-    if (this._ended) {
+    if (this.#ended) {
       throw new Error('called end multiple times');
     }
     if (typeof operations !== 'number') {
@@ -206,7 +244,7 @@ class Benchmark {
       elapsed[1] = 1;
     }
 
-    this._ended = true;
+    this.#ended = true;
     const time = elapsed[0] + elapsed[1] / 1e9;
     const rate = operations / time;
     this.report(rate, elapsed);
@@ -216,7 +254,7 @@ class Benchmark {
     sendResult({
       name: this.name,
       conf: this.config,
-      rate: rate,
+      rate,
       time: elapsed[0] + elapsed[1] / 1e9,
       type: 'report',
     });
@@ -334,6 +372,7 @@ function bakeUrlData(type, e = 0, withBase = false, asUrl = false) {
 }
 
 module.exports = {
+  Benchmark,
   PORT: http_benchmarkers.PORT,
   bakeUrlData,
   binding(bindingName) {
@@ -349,8 +388,6 @@ module.exports = {
   createBenchmark(fn, configs, options) {
     return new Benchmark(fn, configs, options);
   },
-  // Benchmark an http server.
-  default_http_benchmarker: http_benchmarkers.default_http_benchmarker,
   sendResult,
   searchParams,
   urlDataTypes: Object.keys(urls).concat(['wpt']),

benchmark/compare.js

@@ -9,7 +9,7 @@ const BenchmarkProgress = require('./_benchmark_progress.js');
 //
 // Parse arguments
 //
-const cli = CLI(`usage: ./node compare.js [options] [--] <category> ...
+const cli = new CLI(`usage: ./node compare.js [options] [--] <category> ...
   Run each benchmark in the <category> directory many times using two different
   node versions. More than one <category> directory can be specified.
   The output is formatted as csv, which can be processed using for

benchmark/crypto/cipher-stream.js

@@ -7,6 +7,8 @@ const bench = common.createBenchmark(main, {
   type: ['asc', 'utf', 'buf'],
   len: [2, 1024, 102400, 1024 * 1024],
   api: ['legacy', 'stream']
+}, {
+  flags: ['--no-warnings']
 });
 
 function main({ api, cipher, type, len, writes }) {

benchmark/fs/read-stream-throughput.js

@@ -11,19 +11,18 @@ tmpdir.refresh();
 const filename = path.resolve(tmpdir.path,
                               `.removeme-benchmark-garbage-${process.pid}`);
 
-let encodingType, encoding, size, filesize;
-
 const bench = common.createBenchmark(main, {
   encodingType: ['buf', 'asc', 'utf'],
-  filesize: [1000 * 1024 * 1024],
-  size: [1024, 4096, 65535, 1024 * 1024]
+  filesize: [1000 * 1024],
+  highWaterMark: [1024, 4096, 65535, 1024 * 1024],
+  n: 1024
 });
 
 function main(conf) {
-  encodingType = conf.encodingType;
-  size = conf.size;
-  filesize = conf.filesize;
+  const { encodingType, highWaterMark, filesize } = conf;
+  let { n } = conf;
 
+  let encoding = '';
   switch (encodingType) {
     case 'buf':
       encoding = null;
@@ -38,34 +37,8 @@ function main(conf) {
       throw new Error(`invalid encodingType: ${encodingType}`);
   }
 
-  makeFile();
-}
-
-function runTest() {
-  assert(fs.statSync(filename).size === filesize);
-  const rs = fs.createReadStream(filename, {
-    highWaterMark: size,
-    encoding: encoding
-  });
-
-  rs.on('open', () => {
-    bench.start();
-  });
-
-  let bytes = 0;
-  rs.on('data', (chunk) => {
-    bytes += chunk.length;
-  });
-
-  rs.on('end', () => {
-    try { fs.unlinkSync(filename); } catch {}
-    // MB/sec
-    bench.end(bytes / (1024 * 1024));
-  });
-}
-
-function makeFile() {
-  const buf = Buffer.allocUnsafe(filesize / 1024);
+  // Make file
+  const buf = Buffer.allocUnsafe(filesize);
   if (encoding === 'utf8') {
     // ü
     for (let i = 0; i < buf.length; i++) {
@@ -78,16 +51,38 @@ function makeFile() {
   }
 
   try { fs.unlinkSync(filename); } catch {}
-  let w = 1024;
   const ws = fs.createWriteStream(filename);
-  ws.on('close', runTest);
+  ws.on('close', runTest.bind(null, filesize, highWaterMark, encoding, n));
   ws.on('drain', write);
   write();
   function write() {
     do {
-      w--;
-    } while (false !== ws.write(buf) && w > 0);
-    if (w === 0)
+      n--;
+    } while (false !== ws.write(buf) && n > 0);
+    if (n === 0)
       ws.end();
   }
 }
+
+function runTest(filesize, highWaterMark, encoding, n) {
+  assert(fs.statSync(filename).size === filesize * n);
+  const rs = fs.createReadStream(filename, {
+    highWaterMark,
+    encoding
+  });
+
+  rs.on('open', () => {
+    bench.start();
+  });
+
+  let bytes = 0;
+  rs.on('data', (chunk) => {
+    bytes += chunk.length;
+  });
+
+  rs.on('end', () => {
+    try { fs.unlinkSync(filename); } catch {}
+    // MB/sec
+    bench.end(bytes / (1024 * 1024));
+  });
+}

benchmark/fs/readfile.js

@@ -14,12 +14,12 @@ const filename = path.resolve(tmpdir.path,
                               `.removeme-benchmark-garbage-${process.pid}`);
 
 const bench = common.createBenchmark(main, {
-  dur: [5],
+  duration: [5],
   len: [1024, 16 * 1024 * 1024],
   concurrent: [1, 10]
 });
 
-function main({ len, dur, concurrent }) {
+function main({ len, duration, concurrent }) {
   try { fs.unlinkSync(filename); } catch {}
   let data = Buffer.alloc(len, 'x');
   fs.writeFileSync(filename, data);
@@ -33,7 +33,7 @@ function main({ len, dur, concurrent }) {
     bench.end(reads);
     try { fs.unlinkSync(filename); } catch {}
     process.exit(0);
-  }, dur * 1000);
+  }, duration * 1000);
 
   function read() {
     fs.readFile(filename, afterRead);

benchmark/http/chunked.js

@@ -13,10 +13,11 @@ const common = require('../common.js');
 const bench = common.createBenchmark(main, {
   n: [1, 4, 8, 16],
   len: [1, 64, 256],
-  c: [100]
+  c: [100],
+  duration: 5
 });
 
-function main({ len, n, c }) {
+function main({ len, n, c, duration }) {
   const http = require('http');
   const chunk = Buffer.alloc(len, '8');
 
@@ -33,7 +34,8 @@ function main({ len, n, c }) {
 
   server.listen(common.PORT, () => {
     bench.http({
-      connections: c
+      connections: c,
+      duration
     }, () => {
       server.close();
     });

benchmark/http/cluster.js

@@ -9,14 +9,15 @@ if (cluster.isMaster) {
     // Unicode confuses ab on os x.
     type: ['bytes', 'buffer'],
     len: [4, 1024, 102400],
-    c: [50, 500]
+    c: [50, 500],
+    duration: 5,
   });
 } else {
   const port = parseInt(process.env.PORT || PORT);
   require('../fixtures/simple-http-server.js').listen(port);
 }
 
-function main({ type, len, c }) {
+function main({ type, len, c, duration }) {
   process.env.PORT = PORT;
   let workers = 0;
   const w1 = cluster.fork();
@@ -32,7 +33,8 @@ function main({ type, len, c }) {
 
       bench.http({
         path: path,
-        connections: c
+        connections: c,
+        duration
       }, () => {
         w1.destroy();
         w2.destroy();

benchmark/http/end-vs-write-end.js

@@ -14,10 +14,11 @@ const bench = common.createBenchmark(main, {
   type: ['asc', 'utf', 'buf'],
   len: [64 * 1024, 128 * 1024, 256 * 1024, 1024 * 1024],
   c: [100],
-  method: ['write', 'end']
+  method: ['write', 'end'],
+  duration: 5
 });
 
-function main({ len, type, method, c }) {
+function main({ len, type, method, c, duration }) {
   const http = require('http');
   let chunk;
   switch (type) {
@@ -49,7 +50,8 @@ function main({ len, type, method, c }) {
 
   server.listen(common.PORT, () => {
     bench.http({
-      connections: c
+      connections: c,
+      duration
     }, () => {
       server.close();
     });

benchmark/http/headers.js

@@ -6,9 +6,10 @@ const http = require('http');
 const bench = common.createBenchmark(main, {
   n: [10, 1000],
   len: [1, 100],
+  duration: 5
 });
 
-function main({ len, n }) {
+function main({ len, n, duration }) {
   const headers = {
     'Connection': 'keep-alive',
     'Transfer-Encoding': 'chunked',
@@ -29,7 +30,8 @@ function main({ len, n }) {
   server.listen(common.PORT, () => {
     bench.http({
       path: '/',
-      connections: 10
+      connections: 10,
+      duration
     }, () => {
       server.close();
     });

benchmark/http/incoming_headers.js

@@ -3,12 +3,13 @@ const common = require('../common.js');
 const http = require('http');
 
 const bench = common.createBenchmark(main, {
-  c: [50],     // Concurrent connections
-  n: [20],     // Number of header lines to append after the common headers
-  w: [0, 6],   // Amount of trailing whitespace
+  connections: [50], // Concurrent connections
+  headers: [20], // Number of header lines to append after the common headers
+  w: [0, 6], // Amount of trailing whitespace
+  duration: 5
 });
 
-function main({ c, n, w }) {
+function main({ connections, headers, w, duration }) {
   const server = http.createServer((req, res) => {
     res.end();
   });
@@ -21,7 +22,7 @@ function main({ c, n, w }) {
       'Date': new Date().toString(),
       'Cache-Control': 'no-cache'
     };
-    for (let i = 0; i < n; i++) {
+    for (let i = 0; i < headers; i++) {
       // Note:
       // - autocannon does not send header values with OWS
       // - wrk can only send trailing OWS. This is a side-effect of wrk
@@ -31,8 +32,9 @@ function main({ c, n, w }) {
     }
     bench.http({
       path: '/',
-      connections: c,
-      headers
+      connections,
+      headers,
+      duration
     }, () => {
       server.close();
     });

benchmark/http/set-header.js

@@ -3,7 +3,8 @@ const common = require('../common.js');
 const PORT = common.PORT;
 
 const bench = common.createBenchmark(main, {
-  res: ['normal', 'setHeader', 'setHeaderWH']
+  res: ['normal', 'setHeader', 'setHeaderWH'],
+  duration: 5
 });
 
 const type = 'bytes';
@@ -15,16 +16,17 @@ const c = 50;
 // normal: writeHead(status, {...})
 // setHeader: statusCode = status, setHeader(...) x2
 // setHeaderWH: setHeader(...), writeHead(status, ...)
-function main({ res }) {
+function main({ res, duration }) {
   process.env.PORT = PORT;
   const server = require('../fixtures/simple-http-server.js')
   .listen(PORT)
   .on('listening', () => {
-    const path = `/${type}/${len}/${chunks}/normal/${chunkedEnc}`;
+    const path = `/${type}/${len}/${chunks}/${res}/${chunkedEnc}`;
 
     bench.http({
       path: path,
-      connections: c
+      connections: c,
+      duration
     }, () => {
       server.close();
     });

benchmark/http/simple.js

@@ -7,18 +7,20 @@ const bench = common.createBenchmark(main, {
   len: [4, 1024, 102400],
   chunks: [1, 4],
   c: [50, 500],
-  chunkedEnc: [1, 0]
+  chunkedEnc: [1, 0],
+  duration: 5
 });
 
-function main({ type, len, chunks, c, chunkedEnc, res }) {
+function main({ type, len, chunks, c, chunkedEnc, duration }) {
   const server = require('../fixtures/simple-http-server.js')
   .listen(common.PORT)
   .on('listening', () => {
     const path = `/${type}/${len}/${chunks}/normal/${chunkedEnc}`;
 
     bench.http({
-      path: path,
-      connections: c
+      path,
+      connections: c,
+      duration
     }, () => {
       server.close();
     });

benchmark/http2/compat.js

@@ -9,10 +9,11 @@ const bench = common.createBenchmark(main, {
   requests: [100, 1000, 5000],
   streams: [1, 10, 20, 40, 100, 200],
   clients: [2],
-  benchmarker: ['h2load']
+  benchmarker: ['test-double-http2'],
+  duration: 5
 }, { flags: ['--no-warnings'] });
 
-function main({ requests, streams, clients }) {
+function main({ requests, streams, clients, duration }) {
   const http2 = require('http2');
   const server = http2.createServer();
   server.on('request', (req, res) => {
@@ -29,7 +30,8 @@ function main({ requests, streams, clients }) {
       requests,
       maxConcurrentStreams: streams,
       clients,
-      threads: clients
+      threads: clients,
+      duration
     }, () => { server.close(); });
   });
 }

benchmark/http2/respond-with-fd.js

@@ -10,10 +10,11 @@ const bench = common.createBenchmark(main, {
   requests: [100, 1000, 5000],
   streams: [1, 10, 20, 40, 100, 200],
   clients: [2],
-  benchmarker: ['h2load']
+  benchmarker: ['test-double-http2'],
+  duration: 5
 }, { flags: ['--no-warnings'] });
 
-function main({ requests, streams, clients }) {
+function main({ requests, streams, clients, duration }) {
   fs.open(file, 'r', (err, fd) => {
     if (err)
       throw err;
@@ -30,6 +31,7 @@ function main({ requests, streams, clients }) {
         requests,
         maxConcurrentStreams: streams,
         clients,
+        duration,
         threads: clients
       }, () => server.close());
     });

benchmark/http2/simple.js

@@ -9,10 +9,11 @@ const bench = common.createBenchmark(main, {
   requests: [100, 1000, 5000],
   streams: [1, 10, 20, 40, 100, 200],
   clients: [2],
-  benchmarker: ['h2load']
+  benchmarker: ['test-double-http2'],
+  duration: 5
 }, { flags: ['--no-warnings'] });
 
-function main({ requests, streams, clients }) {
+function main({ requests, streams, clients, duration }) {
   const http2 = require('http2');
   const server = http2.createServer();
   server.on('stream', (stream) => {
@@ -27,6 +28,7 @@ function main({ requests, streams, clients }) {
       requests,
       maxConcurrentStreams: streams,
       clients,
+      duration,
       threads: clients
     }, () => { server.close(); });
   });

benchmark/http2/write.js

@@ -6,10 +6,11 @@ const bench = common.createBenchmark(main, {
   streams: [100, 200, 1000],
   length: [64 * 1024, 128 * 1024, 256 * 1024, 1024 * 1024],
   size: [100000],
-  benchmarker: ['h2load']
+  benchmarker: ['test-double-http2'],
+  duration: 5
 }, { flags: ['--no-warnings'] });
 
-function main({ streams, length, size }) {
+function main({ streams, length, size, duration }) {
   const http2 = require('http2');
   const server = http2.createServer();
   server.on('stream', (stream) => {
@@ -29,6 +30,7 @@ function main({ streams, length, size }) {
     bench.http({
       path: '/',
       requests: 10000,
+      duration,
       maxConcurrentStreams: streams,
     }, () => { server.close(); });
   });

benchmark/misc/trace.js

@@ -6,7 +6,11 @@ const bench = common.createBenchmark(main, {
   n: [100000],
   method: ['trace', 'isTraceCategoryEnabled']
 }, {
-  flags: ['--expose-internals', '--trace-event-categories', 'foo']
+  flags: [
+    '--expose-internals',
+    '--no-warnings',
+    '--trace-event-categories', 'foo',
+  ]
 });
 
 const {

benchmark/net/net-c2s.js

@@ -9,6 +9,8 @@ const bench = common.createBenchmark(main, {
   len: [64, 102400, 1024 * 1024 * 16],
   type: ['utf', 'asc', 'buf'],
   dur: [5],
+}, {
+  test: { len: 1024 }
 });
 
 let chunk;

benchmark/net/net-pipe.js

@@ -9,6 +9,8 @@ const bench = common.createBenchmark(main, {
   len: [2, 64, 102400, 1024 * 1024 * 16],
   type: ['utf', 'asc', 'buf'],
   dur: [5],
+}, {
+  test: { len: 1024 }
 });
 
 let chunk;

benchmark/net/net-s2c.js

@@ -10,6 +10,8 @@ const bench = common.createBenchmark(main, {
   recvbuflen: [0, 64 * 1024, 1024 * 1024],
   recvbufgenfn: ['true', 'false'],
   dur: [5]
+}, {
+  test: { sendchunklen: 256 }
 });
 
 let chunk;

benchmark/net/net-wrap-js-stream-passthrough.js

@@ -9,6 +9,7 @@ const bench = common.createBenchmark(main, {
   type: ['utf', 'asc', 'buf'],
   dur: [5],
 }, {
+  test: { len: 64 },
   flags: ['--expose-internals']
 });
 

benchmark/net/tcp-raw-c2s.js

@@ -12,7 +12,10 @@ const bench = common.createBenchmark(main, {
   len: [102400, 1024 * 1024 * 16],
   type: ['utf', 'asc', 'buf'],
   dur: [5]
-}, { flags: [ '--expose-internals', '--no-warnings' ] });
+}, {
+  test: { len: 1024 },
+  flags: [ '--expose-internals', '--no-warnings' ]
+});
 
 function main({ dur, len, type }) {
   const {

benchmark/net/tcp-raw-pipe.js

@@ -13,6 +13,7 @@ const bench = common.createBenchmark(main, {
   type: ['utf', 'asc', 'buf'],
   dur: [5]
 }, {
+  test: { len: 1024 },
   flags: [ '--expose-internals', '--no-warnings' ]
 });
 

benchmark/net/tcp-raw-s2c.js

@@ -13,6 +13,7 @@ const bench = common.createBenchmark(main, {
   type: ['utf', 'asc', 'buf'],
   dur: [5]
 }, {
+  test: { len: 1024 },
   flags: [ '--expose-internals', '--no-warnings' ]
 });
 

benchmark/run.js

@@ -4,7 +4,7 @@ const path = require('path');
 const fork = require('child_process').fork;
 const CLI = require('./_cli.js');
 
-const cli = CLI(`usage: ./node run.js [options] [--] <category> ...
+const cli = new CLI(`usage: ./node run.js [options] [--] <category> ...
   Run each benchmark in the <category> directory a single time, more than one
   <category> directory can be specified.
 
@@ -14,6 +14,9 @@ const cli = CLI(`usage: ./node run.js [options] [--] <category> ...
                             repeated)
   --set    variable=value   set benchmark variable (can be repeated)
   --format [simple|csv]     optional value that specifies the output format
+  test                      only run a single configuration from the options
+                            matrix
+  all                       each benchmark category is run one after the other
 `, { arrayArgs: ['set', 'filter', 'exclude'] });
 const benchmarks = cli.benchmarks();
 
@@ -37,7 +40,11 @@ if (format === 'csv') {
 
 (function recursive(i) {
   const filename = benchmarks[i];
-  const child = fork(path.resolve(__dirname, filename), cli.optional.set);
+  const child = fork(
+    path.resolve(__dirname, filename),
+    cli.test ? ['--test'] : [],
+    cli.optional.set
+  );
 
   if (format !== 'csv') {
     console.log();
@@ -51,10 +58,10 @@ if (format === 'csv') {
     // Construct configuration string, " A=a, B=b, ..."
     let conf = '';
     for (const key of Object.keys(data.conf)) {
-      conf += ` ${key}=${JSON.stringify(data.conf[key])}`;
+      if (conf !== '')
+        conf += ' ';
+      conf += `${key}=${JSON.stringify(data.conf[key])}`;
     }
-    // Delete first space of the configuration
-    conf = conf.slice(1);
     if (format === 'csv') {
       // Escape quotes (") for correct csv formatting
       conf = conf.replace(/"/g, '""');

benchmark/scatter.js

@@ -7,7 +7,7 @@ const CLI = require('./_cli.js');
 //
 // Parse arguments
 //
-const cli = CLI(`usage: ./node scatter.js [options] [--] <filename>
+const cli = new CLI(`usage: ./node scatter.js [options] [--] <filename>
   Run the benchmark script <filename> many times and output the rate (ops/s)
   together with the benchmark variables as a csv.
 

benchmark/tls/secure-pair.js

@@ -4,6 +4,8 @@ const bench = common.createBenchmark(main, {
   dur: [5],
   securing: ['SecurePair', 'TLSSocket', 'clear'],
   size: [100, 1024, 1024 * 1024]
+}, {
+  flags: ['--no-warnings']
 });
 
 const fixtures = require('../../test/common/fixtures');

benchmark/util/type-check.js

@@ -31,7 +31,7 @@ const bench = common.createBenchmark(main, {
   argument: ['true', 'false-primitive', 'false-object'],
   n: [1e5]
 }, {
-  flags: ['--expose-internals']
+  flags: ['--expose-internals', '--no-warnings']
 });
 
 function main({ type, argument, version, n }) {

benchmark/zlib/pipe.js

@@ -8,6 +8,11 @@ const bench = common.createBenchmark(main, {
   duration: [5],
   type: ['string', 'buffer'],
   algorithm: ['gzip', 'brotli']
+}, {
+  test: {
+    inputLen: 1024,
+    duration: 0.2
+  }
 });
 
 function main({ inputLen, duration, type, algorithm }) {

doc/guides/writing-and-running-benchmarks.md

@@ -519,7 +519,8 @@ const common = require('../common.js');
 
 const bench = common.createBenchmark(main, {
   kb: [64, 128, 256, 1024],
-  connections: [100, 500]
+  connections: [100, 500],
+  duration: 5
 });
 
 function main(conf) {
@@ -546,8 +547,8 @@ Supported options keys are:
 * `path` - defaults to `/`
 * `connections` - number of concurrent connections to use, defaults to 100
 * `duration` - duration of the benchmark in seconds, defaults to 10
-* `benchmarker` - benchmarker to use, defaults to
-`common.default_http_benchmarker`
+* `benchmarker` - benchmarker to use, defaults to the first available http
+  benchmarker
 
 [autocannon]: https://github.com/mcollina/autocannon
 [wrk]: https://github.com/wg/wrk

test/common/benchmark.js

@@ -8,13 +8,8 @@ const path = require('path');
 
 const runjs = path.join(__dirname, '..', '..', 'benchmark', 'run.js');
 
-function runBenchmark(name, args, env) {
-  const argv = [];
-
-  for (let i = 0; i < args.length; i++) {
-    argv.push('--set');
-    argv.push(args[i]);
-  }
+function runBenchmark(name, env) {
+  const argv = ['test'];
 
   argv.push(name);