coverage: expose native V8 coverage

native V8 coverage reports can now be written to disk by setting the
variable NODE_V8_COVERAGE=dir

PR-URL: #22527
Reviewed-By: Anna Henningsen <anna@addaleax.net>
Reviewed-By: Yang Guo <yangguo@chromium.org>
Reviewed-By: Ruben Bridgewater <ruben@bridgewater.de>
Reviewed-By: Evan Lucas <evanlucas@me.com>
Reviewed-By: Rod Vagg <rod@vagg.org>

doc/api/cli.md

@@ -625,6 +625,32 @@ Path to the file used to store the persistent REPL history. The default path is
 `~/.node_repl_history`, which is overridden by this variable. Setting the value
 to an empty string (`''` or `' '`) disables persistent REPL history.
 
+### `NODE_V8_COVERAGE=dir`
+
+When set, Node.js will begin outputting [V8 JavaScript code coverage][] to the
+directory provided as an argument. Coverage is output as an array of
+[ScriptCoverage][] objects:
+
+```json
+{
+  "result": [
+    {
+      "scriptId": "67",
+      "url": "internal/tty.js",
+      "functions": []
+    }
+  ]
+}
+```
+
+`NODE_V8_COVERAGE` will automatically propagate to subprocesses, making it
+easier to instrument applications that call the `child_process.spawn()` family
+of functions. `NODE_V8_COVERAGE` can be set to an empty string, to prevent
+propagation.
+
+At this time coverage is only collected in the main thread and will not be
+output for code executed by worker threads.
+
 ### `OPENSSL_CONF=file`
 <!-- YAML
 added: v6.11.0
@@ -691,6 +717,8 @@ greater than `4` (its current default value). For more information, see the
 [`process.setUncaughtExceptionCaptureCallback()`]: process.html#process_process_setuncaughtexceptioncapturecallback_fn
 [Chrome DevTools Protocol]: https://chromedevtools.github.io/devtools-protocol/
 [REPL]: repl.html
+[ScriptCoverage]: https://chromedevtools.github.io/devtools-protocol/tot/Profiler#type-ScriptCoverage
+[V8 JavaScript code coverage]: https://v8project.blogspot.com/2017/12/javascript-code-coverage.html
 [debugger]: debugger.html
 [emit_warning]: process.html#process_process_emitwarning_warning_type_code_ctor
 [experimental ECMAScript Module]: esm.html#esm_loader_hooks

lib/child_process.js

@@ -496,6 +496,14 @@ function normalizeSpawnArguments(file, args, options) {
   var env = options.env || process.env;
   var envPairs = [];
 
+  // process.env.NODE_V8_COVERAGE always propagates, making it possible to
+  // collect coverage for programs that spawn with white-listed environment.
+  if (process.env.NODE_V8_COVERAGE &&
+      !Object.prototype.hasOwnProperty.call(options.env || {},
+                                            'NODE_V8_COVERAGE')) {
+    env.NODE_V8_COVERAGE = process.env.NODE_V8_COVERAGE;
+  }
+
   // Prototype values are intentionally included.
   for (var key in env) {
     const value = env[key];

lib/internal/bootstrap/node.js

@@ -98,6 +98,12 @@
     if (global.__coverage__)
       NativeModule.require('internal/process/write-coverage').setup();
 
+    if (process.env.NODE_V8_COVERAGE) {
+      const { resolve } = NativeModule.require('path');
+      process.env.NODE_V8_COVERAGE = resolve(process.env.NODE_V8_COVERAGE);
+      NativeModule.require('internal/process/coverage').setup();
+    }
+
 
     {
       const traceEvents = process.binding('trace_events');

lib/internal/print_help.js

@@ -29,6 +29,8 @@ const envVars = new Map([
     'of stderr' }],
   ['NODE_REPL_HISTORY', { helpText: 'path to the persistent REPL ' +
     'history file' }],
+  ['NODE_V8_COVERAGE', { helpText: 'directory to output v8 coverage JSON ' +
+    'to' }],
   ['OPENSSL_CONF', { helpText: 'load OpenSSL configuration from file' }]
 ].concat(hasIntl ? [
   ['NODE_ICU_DATA', { helpText: 'data path for ICU (Intl object) data' +

lib/internal/process/coverage.js

@@ -0,0 +1,71 @@
+'use strict';
+const path = require('path');
+const { mkdirSync, writeFileSync } = require('fs');
+// TODO(addaleax): add support for coverage to worker threads.
+const hasInspector = process.config.variables.v8_enable_inspector === 1 &&
+  require('internal/worker').isMainThread;
+let inspector = null;
+if (hasInspector) inspector = require('inspector');
+
+let session;
+
+function writeCoverage() {
+  if (!session) {
+    return;
+  }
+
+  const filename = `coverage-${process.pid}-${Date.now()}.json`;
+  try {
+    // TODO(bcoe): switch to mkdirp once #22302 is addressed.
+    mkdirSync(process.env.NODE_V8_COVERAGE);
+  } catch (err) {
+    if (err.code !== 'EEXIST') {
+      console.error(err);
+      return;
+    }
+  }
+
+  const target = path.join(process.env.NODE_V8_COVERAGE, filename);
+
+  try {
+    session.post('Profiler.takePreciseCoverage', (err, coverageInfo) => {
+      if (err) return console.error(err);
+      try {
+        writeFileSync(target, JSON.stringify(coverageInfo));
+      } catch (err) {
+        console.error(err);
+      }
+    });
+  } catch (err) {
+    console.error(err);
+  } finally {
+    session.disconnect();
+    session = null;
+  }
+}
+
+exports.writeCoverage = writeCoverage;
+
+function setup() {
+  if (!hasInspector) {
+    console.warn('coverage currently only supported in main thread');
+    return;
+  }
+
+  session = new inspector.Session();
+  session.connect();
+  session.post('Profiler.enable');
+  session.post('Profiler.startPreciseCoverage', { callCount: true,
+                                                  detailed: true });
+
+  const reallyReallyExit = process.reallyExit;
+
+  process.reallyExit = function(code) {
+    writeCoverage();
+    reallyReallyExit(code);
+  };
+
+  process.on('exit', writeCoverage);
+}
+
+exports.setup = setup;

lib/internal/process/per_thread.js

@@ -172,6 +172,10 @@ function setupKillAndExit() {
 
   process.kill = function(pid, sig) {
     var err;
+    if (process.env.NODE_V8_COVERAGE) {
+      const { writeCoverage } = require('internal/process/coverage');
+      writeCoverage();
+    }
 
     // eslint-disable-next-line eqeqeq
     if (pid != (pid | 0)) {

node.gyp

@@ -143,6 +143,7 @@
       'lib/internal/process/worker_thread_only.js',
       'lib/internal/querystring.js',
       'lib/internal/process/write-coverage.js',
+      'lib/internal/process/coverage.js',
       'lib/internal/readline.js',
       'lib/internal/repl.js',
       'lib/internal/repl/await.js',

test/fixtures/v8-coverage/basic.js

@@ -0,0 +1,6 @@
+const a = 99;
+if (true) {
+  const b = 101;
+} else {
+  const c = 102;
+}

test/fixtures/v8-coverage/exit-1.js

@@ -0,0 +1,7 @@
+const a = 99;
+if (true) {
+  const b = 101;
+} else {
+  const c = 102;
+}
+process.exit(1);

test/fixtures/v8-coverage/sigint.js

@@ -0,0 +1,7 @@
+const a = 99;
+if (true) {
+  const b = 101;
+} else {
+  const c = 102;
+}
+process.kill(process.pid, "SIGINT");

test/fixtures/v8-coverage/spawn-subprocess-no-cov.js

@@ -0,0 +1,5 @@
+const { spawnSync } = require('child_process');
+const env = Object.assign({}, process.env, { NODE_V8_COVERAGE: '' });
+spawnSync(process.execPath, [require.resolve('./subprocess')], {
+  env: env
+});

test/fixtures/v8-coverage/spawn-subprocess.js

@@ -0,0 +1,6 @@
+const { spawnSync } = require('child_process');
+const env = Object.assign({}, process.env);
+delete env.NODE_V8_COVERAGE
+spawnSync(process.execPath, [require.resolve('./subprocess')], {
+  env: env
+});

test/fixtures/v8-coverage/subprocess.js

@@ -0,0 +1,8 @@
+const a = 99;
+setTimeout(() => {
+  if (false) {
+    const b = 101;
+  } else if (false) {
+    const c = 102;
+  }
+}, 10);

test/parallel/test-v8-coverage.js

@@ -0,0 +1,105 @@
+'use strict';
+
+if (!process.config.variables.v8_enable_inspector) return;
+
+const common = require('../common');
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+const { spawnSync } = require('child_process');
+
+const tmpdir = require('../common/tmpdir');
+tmpdir.refresh();
+
+let dirc = 0;
+function nextdir() {
+  return `cov_${++dirc}`;
+}
+
+// outputs coverage when event loop is drained, with no async logic.
+{
+  const coverageDirectory = path.join(tmpdir.path, nextdir());
+  const output = spawnSync(process.execPath, [
+    require.resolve('../fixtures/v8-coverage/basic')
+  ], { env: { ...process.env, NODE_V8_COVERAGE: coverageDirectory } });
+  assert.strictEqual(output.status, 0);
+  const fixtureCoverage = getFixtureCoverage('basic.js', coverageDirectory);
+  assert.ok(fixtureCoverage);
+  // first branch executed.
+  assert.strictEqual(fixtureCoverage.functions[1].ranges[0].count, 1);
+  // second branch did not execute.
+  assert.strictEqual(fixtureCoverage.functions[1].ranges[1].count, 0);
+}
+
+// outputs coverage when process.exit(1) exits process.
+{
+  const coverageDirectory = path.join(tmpdir.path, nextdir());
+  const output = spawnSync(process.execPath, [
+    require.resolve('../fixtures/v8-coverage/exit-1')
+  ], { env: { ...process.env, NODE_V8_COVERAGE: coverageDirectory } });
+  assert.strictEqual(output.status, 1);
+  const fixtureCoverage = getFixtureCoverage('exit-1.js', coverageDirectory);
+  assert.ok(fixtureCoverage, 'coverage not found for file');
+  // first branch executed.
+  assert.strictEqual(fixtureCoverage.functions[1].ranges[0].count, 1);
+  // second branch did not execute.
+  assert.strictEqual(fixtureCoverage.functions[1].ranges[1].count, 0);
+}
+
+// outputs coverage when process.kill(process.pid, "SIGINT"); exits process.
+{
+  const coverageDirectory = path.join(tmpdir.path, nextdir());
+  const output = spawnSync(process.execPath, [
+    require.resolve('../fixtures/v8-coverage/sigint')
+  ], { env: { ...process.env, NODE_V8_COVERAGE: coverageDirectory } });
+  if (!common.isWindows) {
+    assert.strictEqual(output.signal, 'SIGINT');
+  }
+  const fixtureCoverage = getFixtureCoverage('sigint.js', coverageDirectory);
+  assert.ok(fixtureCoverage);
+  // first branch executed.
+  assert.strictEqual(fixtureCoverage.functions[1].ranges[0].count, 1);
+  // second branch did not execute.
+  assert.strictEqual(fixtureCoverage.functions[1].ranges[1].count, 0);
+}
+
+// outputs coverage from subprocess.
+{
+  const coverageDirectory = path.join(tmpdir.path, nextdir());
+  const output = spawnSync(process.execPath, [
+    require.resolve('../fixtures/v8-coverage/spawn-subprocess')
+  ], { env: { ...process.env, NODE_V8_COVERAGE: coverageDirectory } });
+  assert.strictEqual(output.status, 0);
+  const fixtureCoverage = getFixtureCoverage('subprocess.js',
+                                             coverageDirectory);
+  assert.ok(fixtureCoverage);
+  // first branch executed.
+  assert.strictEqual(fixtureCoverage.functions[2].ranges[0].count, 1);
+  // second branch did not execute.
+  assert.strictEqual(fixtureCoverage.functions[2].ranges[1].count, 0);
+}
+
+// does not output coverage if NODE_V8_COVERAGE is empty.
+{
+  const coverageDirectory = path.join(tmpdir.path, nextdir());
+  const output = spawnSync(process.execPath, [
+    require.resolve('../fixtures/v8-coverage/spawn-subprocess-no-cov')
+  ], { env: { ...process.env, NODE_V8_COVERAGE: coverageDirectory } });
+  assert.strictEqual(output.status, 0);
+  const fixtureCoverage = getFixtureCoverage('subprocess.js',
+                                             coverageDirectory);
+  assert.strictEqual(fixtureCoverage, undefined);
+}
+
+// extracts the coverage object for a given fixture name.
+function getFixtureCoverage(fixtureFile, coverageDirectory) {
+  const coverageFiles = fs.readdirSync(coverageDirectory);
+  for (const coverageFile of coverageFiles) {
+    const coverage = require(path.join(coverageDirectory, coverageFile));
+    for (const fixtureCoverage of coverage.result) {
+      if (fixtureCoverage.url.indexOf(`${path.sep}${fixtureFile}`) !== -1) {
+        return fixtureCoverage;
+      }
+    }
+  }
+}