chore: include Node.js driver and devtools-connect in snapshot MONGOS…

…H-1727 (#1846)

Bring back `@mongodb-js/devtools-connect` into the list of snapshotted
dependencies.

To achieve this, replace the built-in `http`, `https`, `tls` and
`child_process` modules with modules which lazy-load the real modules
when they are actually being used.

config/webpack.base.config.js

@@ -27,21 +27,6 @@ module.exports = {
 
   externals: {
     "node:crypto": "commonjs2 crypto",
-    // node-fetch https://github.com/node-fetch/node-fetch/blob/8b3320d2a7c07bce4afc6b2bf6c3bbddda85b01f/src/index.js#L9
-    "node:buffer": "commonjs2 buffer",
-    "node:http": "commonjs2 http",
-    "node:https": "commonjs2 https",
-    "node:stream": "commonjs2 stream",
-    "node:zlib": "commonjs2 zlib", 
-    "node:url": "commonjs2 url",
-    "node:util": "commonjs2 util",
-    "node:events": "commonjs2 events",
-    "node:net": "commonjs2 net",
-    // https://github.com/node-fetch/node-fetch/blob/8b3320d2a7c07bce4afc6b2bf6c3bbddda85b01f/test/main.js#L2
-    "node:dns": "commonjs2 dns",
-    "node:fs": "commonjs2 fs",
-    "node:path": "commonjs2 path",
-    "node:vm": "commonjs2 vm",
     electron: "commonjs2 electron" // optional dep of the OIDC plugin
   },
 

packages/build/src/compile/signable-compiler.ts

@@ -184,7 +184,7 @@ export class SignableCompiler {
       useNodeSnapshot: true,
       // To account for the fact that we are manually patching Node.js to include
       // https://github.com/nodejs/node/pull/50453 until we have caught up with upstream
-      nodeSnapshotConfigFlags: ['WithoutCodeCache'],
+      nodeSnapshotConfigFlags: ['Default'],
     });
   }
 }

packages/cli-repl/webpack.config.js

@@ -1,12 +1,23 @@
 'use strict';
 const fs = require('fs');
+const Module = require('module');
 const crypto = require('crypto');
 const { merge } = require('webpack-merge');
 const path = require('path');
 const { WebpackDependenciesPlugin } = require('@mongodb-js/sbom-tools');
 
 const baseWebpackConfig = require('../../config/webpack.base.config');
 
+// Builtins that the driver and/or devtools-connect refer to but which
+// cannot be snapshotted yet
+// https://github.com/nodejs/node/pull/50943 addresses some of this,
+// we can try to remove at least child_process once we are using a
+// Node.js version that supports it.
+const lazyNodeBuiltins = ['http', 'https', 'tls', 'child_process'];
+const eagerNodeBuiltins = Module.builtinModules.filter(
+  (m) => !lazyNodeBuiltins.includes(m)
+);
+
 const webpackDependenciesPlugin = new WebpackDependenciesPlugin({
   outputFilename: path.resolve(
     __dirname,
@@ -38,8 +49,30 @@ const config = {
       // @babel/code-frame loads chalk loads supports-color which checks
       // for TTY color support during startup rather than at runtime
       '@babel/code-frame': makeLazyForwardModule('@babel/code-frame'),
+      // express is used by oidc-plugin but is a) quite heavy and rarely used
+      // b) uses http, which is not a supported module for snapshots at this point.
+      express: makeLazyForwardModule('express'),
+      'openid-client': makeLazyForwardModule('openid-client'),
+      ...Object.fromEntries(
+        lazyNodeBuiltins.map((m) => [m, makeLazyForwardModule(m)])
+      ),
+      ...Object.fromEntries(
+        lazyNodeBuiltins.map((m) => [`node:${m}`, makeLazyForwardModule(m)])
+      ),
     },
   },
+
+  externals: {
+    electron: 'commonjs2 electron', // optional dep of the OIDC plugin
+    ...Object.fromEntries(eagerNodeBuiltins.map((m) => [m, `commonjs2 ${m}`])),
+    ...Object.fromEntries(
+      Module.builtinModules.map((m) => [`node:${m}`, `commonjs2 ${m}`])
+    ), // node: builtin specifiers need to be always declared as externals in webpack right now
+  },
+
+  externalsPresets: {
+    node: false,
+  },
 };
 
 module.exports = merge(baseWebpackConfig, config);
@@ -57,11 +90,13 @@ function makeLazyForwardModule(pkg) {
     crypto.createHash('sha256').update(pkg).digest('hex').slice(0, 16) + '.js'
   );
 
+  const realRequire = Module.isBuiltin(pkg)
+    ? `__non_webpack_require__(${S(pkg)})`
+    : `require(${S(require.resolve(pkg))})`;
+
   const moduleContents = require(pkg);
   let source = `'use strict';\nlet _cache;\n`;
-  source += `function orig() {\n_cache = require(${S(
-    require.resolve(pkg)
-  )}); orig = () => _cache; return _cache;\n}\n`;
+  source += `function orig() {\n_cache = ${realRequire}; orig = () => _cache; return _cache;\n}\n`;
   if (typeof moduleContents === 'function') {
     source += `module.exports = function(...args) { return orig().apply(this, args); };\n`;
   } else {

packages/logging/src/setup-logger-and-telemetry.ts

@@ -36,6 +36,7 @@ import { inspect } from 'util';
 import type { MongoLogWriter } from 'mongodb-log-writer';
 import { mongoLogId } from 'mongodb-log-writer';
 import type { MongoshAnalytics } from './analytics-helpers';
+import { hookLogger } from '@mongodb-js/devtools-connect';
 
 /**
  * A helper class for keeping track of how often specific events occurred.
@@ -685,7 +686,6 @@ export function setupLoggerAndTelemetry(
   // devtools-connect package which was split out from mongosh.
   // 'mongodb' is not supported in startup snapshots yet.
   // eslint-disable-next-line @typescript-eslint/no-var-requires
-  const { hookLogger } = require('@mongodb-js/devtools-connect');
   hookLogger(bus, log, 'mongosh', redactURICredentials);
 
   bus.on('mongosh-sp:reset-connection-options', function () {

packages/service-provider-server/src/cli-service-provider.ts

@@ -9,7 +9,6 @@ import type {
   RunCursorCommandOptions,
   ClientEncryptionOptions,
   MongoClient,
-  ReadPreference,
   MongoMissingDependencyError,
 } from 'mongodb';
 
@@ -82,12 +81,13 @@ import { EventEmitter } from 'events';
 import type { CreateEncryptedCollectionOptions } from '@mongosh/service-provider-core';
 import type { DevtoolsConnectionState } from '@mongodb-js/devtools-connect';
 import { isDeepStrictEqual } from 'util';
-
-// 'mongodb' is not supported in startup snapshots yet.
-// eslint-disable-next-line @typescript-eslint/consistent-type-imports
-function driver(): typeof import('mongodb') {
-  return require('mongodb');
-}
+import * as driver from 'mongodb';
+import {
+  MongoClient as MongoClientCtor,
+  ReadPreference,
+  ClientEncryption,
+} from 'mongodb';
+import { connectMongoClient } from '@mongodb-js/devtools-connect';
 
 const bsonlib = () => {
   const {
@@ -105,7 +105,7 @@ const bsonlib = () => {
     BSONSymbol,
     BSONRegExp,
     BSON,
-  } = driver();
+  } = driver;
   return {
     Binary,
     Code,
@@ -229,9 +229,6 @@ class CliServiceProvider
       };
     }
 
-    const { MongoClient: MongoClientCtor } = driver();
-    const { connectMongoClient } = require('@mongodb-js/devtools-connect');
-
     let client: MongoClient;
     let state: DevtoolsConnectionState | undefined;
     if (cliOptions.nodb) {
@@ -312,7 +309,7 @@ class CliServiceProvider
     return {
       nodeDriverVersion: tryCall(() => require('mongodb/package.json').version),
       libmongocryptVersion: tryCall(
-        () => driver().ClientEncryption.libmongocryptVersion // getter that actually loads the native addon (!)
+        () => ClientEncryption.libmongocryptVersion // getter that actually loads the native addon (!)
       ),
       libmongocryptNodeBindingsVersion: tryCall(
         () => require('mongodb-client-encryption/package.json').version
@@ -371,8 +368,6 @@ class CliServiceProvider
     connectionString: ConnectionString | string,
     clientOptions: DevtoolsConnectOptions
   ): Promise<{ client: MongoClient; state: DevtoolsConnectionState }> {
-    const { MongoClient: MongoClientCtor } = driver();
-    const { connectMongoClient } = require('@mongodb-js/devtools-connect');
     try {
       return await connectMongoClient(
         connectionString.toString(),
@@ -1302,7 +1297,6 @@ class CliServiceProvider
   readPreferenceFromOptions(
     options?: Omit<ReadPreferenceFromOptions, 'session'>
   ): ReadPreferenceLike | undefined {
-    const { ReadPreference } = driver();
     return ReadPreference.fromOptions(options);
   }
 
@@ -1509,7 +1503,6 @@ class CliServiceProvider
   createClientEncryption(
     options: ClientEncryptionOptions
   ): MongoCryptClientEncryption {
-    const { ClientEncryption } = driver();
     return new ClientEncryption(this.mongoClient, options);
   }
 }