From bc05a5f624b88594daed6f34218abdec5162e382 Mon Sep 17 00:00:00 2001
From: Joyee Cheung <joyeec9h3@gmail.com>
Date: Mon, 5 Dec 2022 11:07:08 +0100
Subject: [PATCH] benchmark: rewrite the startup performance benchmark

The previous benchmark was inaccurate in several ways:

- It previously measured "how many workers/child processes can finish
  loading in parallel in 1 second" which is prone to wrong
  interpretations. It's also not typical for users to spin up >20
  workers/child processes in parallel, which was what the benchmark
  usually ended up doing. The parallelism can also be greatly affected
  by system configurations.
- The time spent on loading a Node.js binary for the first time varies
  greatly depending on system configurations, but it was inevitable
  for the benchmark to load Node.js multiple times to reduce the
  influence of fluctuations.

This patch rewrites the benchmark so that:

- It now measures "how long it takes to finish 30 workers/child processes
  in serial" which is generally more in line with how other benchmarks
  are written and with the figures one gets from doing something like
  `time node index.js` or `hyperfine 'node index.js'`
- It now warms up the loading of the Node.js instance to reduce the
  statistical outliers, so that we get more meaningful figures when
  trying to compare the startup performance of different Node.js
  binaries.
---
 benchmark/misc/startup.js | 90 +++++++++++++++++----------------------
 1 file changed, 39 insertions(+), 51 deletions(-)

diff --git a/benchmark/misc/startup.js b/benchmark/misc/startup.js
index 561c516155bc01..2a1e62922a783f 100644
--- a/benchmark/misc/startup.js
+++ b/benchmark/misc/startup.js
@@ -1,80 +1,68 @@
 'use strict';
 const common = require('../common.js');
-const { spawn } = require('child_process');
+const { spawn, spawnSync } = require('child_process');
 const path = require('path');
 
 let Worker;  // Lazy loaded in main
 
 const bench = common.createBenchmark(main, {
-  dur: [1],
   script: [
     'benchmark/fixtures/require-builtins',
-    'benchmark/fixtures/require-cachable',
     'test/fixtures/semicolon',
   ],
-  mode: ['process', 'worker']
-}, {
-  flags: ['--expose-internals']
+  mode: ['process', 'worker'],
+  count: [30],
 });
 
-function spawnProcess(script) {
+function spawnProcess(script, bench, state) {
   const cmd = process.execPath || process.argv[0];
-  const argv = ['--expose-internals', script];
-  return spawn(cmd, argv);
-}
+  while (state.finished < state.count) {
+    const child = spawnSync(cmd, [script]);
+    if (child.status !== 0) {
+      console.log('---- STDOUT ----');
+      console.log(child.stdout.toString());
+      console.log('---- STDERR ----');
+      console.log(child.stderr.toString());
+      throw new Error(`Child process stopped with exit code ${child.status}`);
+    }
+    state.finished++;
+    if (state.finished === 0) {
+      // Finished warmup.
+      bench.start();
+    }
 
-function spawnWorker(script) {
-  return new Worker(script, { stderr: true, stdout: true });
+    if (state.finished == state.count) {
+      bench.end(state.count);
+    }
+  }
 }
 
-function start(state, script, bench, getNode) {
-  const node = getNode(script);
-  let stdout = '';
-  let stderr = '';
-
-  node.stdout.on('data', (data) => {
-    stdout += data;
-  });
-
-  node.stderr.on('data', (data) => {
-    stderr += data;
-  });
-
-  node.on('exit', (code) => {
+function spawnWorker(script, bench, state) {
+  const child = new Worker(script);
+  child.on('exit', (code) => {
     if (code !== 0) {
-      console.error('------ stdout ------');
-      console.error(stdout);
-      console.error('------ stderr ------');
-      console.error(stderr);
-      throw new Error(`Error during node startup, exit code ${code}`);
+      throw new Error(`Worker stopped with exit code ${code}`);
     }
-    state.throughput++;
-
-    if (state.go) {
-      start(state, script, bench, getNode);
+    state.finished++;
+    if (state.finished === 0) {
+      // Finished warmup.
+      bench.start();
+    }
+    if (state.finished < state.count) {
+      spawnProcess(script, bench, state);
     } else {
-      bench.end(state.throughput);
+      bench.end(state.count);
     }
   });
 }
 
-function main({ dur, script, mode }) {
-  const state = {
-    go: true,
-    throughput: 0
-  };
-
-  require('timers').setTimeout(() => {
-    state.go = false;
-  }, dur * 1000);
-
+function main({ count, script, mode }) {
   script = path.resolve(__dirname, '../../', `${script}.js`);
+  const state = { count, finished: -3 };
   if (mode === 'worker') {
-    Worker = require('worker_threads').Worker;
-    bench.start();
-    start(state, script, bench, spawnWorker);
+    Worker = require('worker_threads').Worker;  // Warm up.
+    spawnWorker(script, bench, state);
   } else {
-    bench.start();
-    start(state, script, bench, spawnProcess);
+    spawnProcess(script, bench, state);
   }
 }