Refactor fs (cache/tracing) layer into self-contained API

.gitignore

@@ -12,3 +12,4 @@ tsconfig.schemastore-schema.json
 /website/static/api
 /tsconfig.tsbuildinfo
 /temp
+/.tmp

development-docs/caches.md

@@ -0,0 +1,32 @@
+Lots of caching in ts-node.
+
+## Caches
+
+### FS cache:
+caches results of primitive ts.sys.readFile, etc operations
+Shared across compiler and config loader
+
+### fileContents (and fileVersions) cache:
+sits in front of fs cache.
+Node.js module loading mechanism reads file contents from disk.  That's put into this cache.
+
+### Output cache:
+Caches the emitted JS syntax from compilation.
+Has appended //# sourcemap comments.
+source-map-support reads from here before fallback to filesystem.
+
+### source-map-support cache:
+caches fs.readFile calls
+overlayed by output cache above
+overlayed by sourcesContents from parsed sourcemaps
+
+### SourceFile cache: (does not exist today)
+for Compiler API codepath
+to avoid re-parsing SourceFile repeatedly
+
+## Questions
+
+If both:
+- source-map-support caches a sourcesContents string of a .ts file
+- cachedFsReader caches the same .ts file from disk
+...which is used by source-map-support?  Does it matter since they should be identical?

development-docs/ts-sys.ts

@@ -0,0 +1,36 @@
+import ts = require("typescript");
+import { getPatternFromSpec } from "../src/ts-internals";
+
+// Notes and demos to understand `ts.sys`
+
+console.dir(ts.sys.getCurrentDirectory());
+// Gets names (not paths) of all directories that are direct children of given path
+// Never throws
+// Accepts trailing `/` or not
+console.dir(ts.sys.getDirectories(ts.sys.getCurrentDirectory()));
+
+/////
+
+// Returns array of absolute paths
+// Never returns directories; only files
+
+// Values can have period or not; are interpreted as a suffix ('o.svg' matches logo.svg; seems to also match if you embed / directory delimiters)
+// [''] is the same as undefined; returns everything
+const extensions: string[] | undefined = [''];
+// Supports wildcards; ts-style globs?
+const exclude: string[] | undefined = undefined;
+const include: string[] | undefined = ['*/????????????'];
+// Depth == 0 is the same as undefined: unlimited depth
+// Depth == 1 is only direct children of directory
+const depth: number | undefined = undefined;
+console.dir(ts.sys.readDirectory(ts.sys.getCurrentDirectory(), extensions, exclude, include, depth));
+
+// To overlay virtual filesystem contents over `ts.sys.readDirectory`, try this:
+// start with array of all virtual files
+// Filter by those having base directory prefix
+// if extensions is array, do an `endsWith` filter
+// if exclude is an array, use `getPatternFromSpec` and filter out anything that matches
+// if include is an array, use `getPatternFromSpec` and filter out anything that does not match at least one
+// if depth is non-zero, count the number of directory delimiters following the base directory prefix
+
+console.log(getPatternFromSpec('foo/*/bar', ts.sys.getCurrentDirectory()));

dist-raw/node-cjs-loader-utils.js

@@ -3,131 +3,136 @@
 // Each function and variable below must have a comment linking to the source in node's github repo.
 
 const path = require('path');
-const packageJsonReader = require('./node-package-json-reader');
 const {JSONParse} = require('./node-primordials');
 const {normalizeSlashes} = require('../dist/util');
 
-module.exports.assertScriptCanLoadAsCJSImpl = assertScriptCanLoadAsCJSImpl;
+/** @param {ReturnType<typeof import('./node-package-json-reader').createNodePackageJsonReader>} packageJsonReader */
+function createNodeCjsLoaderUtils(packageJsonReader) {
 
-/**
- * copied from Module._extensions['.js']
- * https://github.com/nodejs/node/blob/v15.3.0/lib/internal/modules/cjs/loader.js#L1113-L1120
- * @param {import('../src/index').Service} service
- * @param {NodeJS.Module} module
- * @param {string} filename
- */
-function assertScriptCanLoadAsCJSImpl(service, module, filename) {
-  const pkg = readPackageScope(filename);
+  /**
+   * copied from Module._extensions['.js']
+   * https://github.com/nodejs/node/blob/v15.3.0/lib/internal/modules/cjs/loader.js#L1113-L1120
+   * @param {import('../src/index').Service} service
+   * @param {NodeJS.Module} module
+   * @param {string} filename
+   */
+  function assertScriptCanLoadAsCJSImpl(service, module, filename) {
+    const pkg = readPackageScope(filename);
 
-  // ts-node modification: allow our configuration to override
-  const tsNodeClassification = service.moduleTypeClassifier.classifyModule(normalizeSlashes(filename));
-  if(tsNodeClassification.moduleType === 'cjs') return;
+    // ts-node modification: allow our configuration to override
+    const tsNodeClassification = service.moduleTypeClassifier.classifyModule(normalizeSlashes(filename));
+    if(tsNodeClassification.moduleType === 'cjs') return;
 
-  // Function require shouldn't be used in ES modules.
-  if (tsNodeClassification.moduleType === 'esm' || (pkg && pkg.data && pkg.data.type === 'module')) {
-    const parentPath = module.parent && module.parent.filename;
-    const packageJsonPath = pkg ? path.resolve(pkg.path, 'package.json') : null;
-    throw createErrRequireEsm(filename, parentPath, packageJsonPath);
+    // Function require shouldn't be used in ES modules.
+    if (tsNodeClassification.moduleType === 'esm' || (pkg && pkg.data && pkg.data.type === 'module')) {
+      const parentPath = module.parent && module.parent.filename;
+      const packageJsonPath = pkg ? path.resolve(pkg.path, 'package.json') : null;
+      throw createErrRequireEsm(filename, parentPath, packageJsonPath);
+    }
   }
-}
 
-// Copied from https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/modules/cjs/loader.js#L285-L301
-function readPackageScope(checkPath) {
-  const rootSeparatorIndex = checkPath.indexOf(path.sep);
-  let separatorIndex;
-  while (
-    (separatorIndex = checkPath.lastIndexOf(path.sep)) > rootSeparatorIndex
-  ) {
-    checkPath = checkPath.slice(0, separatorIndex);
-    if (checkPath.endsWith(path.sep + 'node_modules'))
-      return false;
-    const pjson = readPackage(checkPath);
-    if (pjson) return {
-      path: checkPath,
-      data: pjson
-    };
+  // Copied from https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/modules/cjs/loader.js#L285-L301
+  function readPackageScope(checkPath) {
+    const rootSeparatorIndex = checkPath.indexOf(path.sep);
+    let separatorIndex;
+    while (
+      (separatorIndex = checkPath.lastIndexOf(path.sep)) > rootSeparatorIndex
+    ) {
+      checkPath = checkPath.slice(0, separatorIndex);
+      if (checkPath.endsWith(path.sep + 'node_modules'))
+        return false;
+      const pjson = readPackage(checkPath);
+      if (pjson) return {
+        path: checkPath,
+        data: pjson
+      };
+    }
+    return false;
   }
-  return false;
-}
 
-// Copied from https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/modules/cjs/loader.js#L249
-const packageJsonCache = new Map();
+  // Copied from https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/modules/cjs/loader.js#L249
+  const packageJsonCache = new Map();
 
-// Copied from https://github.com/nodejs/node/blob/v15.3.0/lib/internal/modules/cjs/loader.js#L275-L304
-function readPackage(requestPath) {
-  const jsonPath = path.resolve(requestPath, 'package.json');
+  // Copied from https://github.com/nodejs/node/blob/v15.3.0/lib/internal/modules/cjs/loader.js#L275-L304
+  function readPackage(requestPath) {
+    const jsonPath = path.resolve(requestPath, 'package.json');
 
-  const existing = packageJsonCache.get(jsonPath);
-  if (existing !== undefined) return existing;
+    const existing = packageJsonCache.get(jsonPath);
+    if (existing !== undefined) return existing;
 
-  const result = packageJsonReader.read(jsonPath);
-  const json = result.containsKeys === false ? '{}' : result.string;
-  if (json === undefined) {
-    packageJsonCache.set(jsonPath, false);
-    return false;
-  }
+    const result = packageJsonReader.read(jsonPath);
+    const json = result.containsKeys === false ? '{}' : result.string;
+    if (json === undefined) {
+      packageJsonCache.set(jsonPath, false);
+      return false;
+    }
 
-  try {
-    const parsed = JSONParse(json);
-    const filtered = {
-      name: parsed.name,
-      main: parsed.main,
-      exports: parsed.exports,
-      imports: parsed.imports,
-      type: parsed.type
-    };
-    packageJsonCache.set(jsonPath, filtered);
-    return filtered;
-  } catch (e) {
-    e.path = jsonPath;
-    e.message = 'Error parsing ' + jsonPath + ': ' + e.message;
-    throw e;
+    try {
+      const parsed = JSONParse(json);
+      const filtered = {
+        name: parsed.name,
+        main: parsed.main,
+        exports: parsed.exports,
+        imports: parsed.imports,
+        type: parsed.type
+      };
+      packageJsonCache.set(jsonPath, filtered);
+      return filtered;
+    } catch (e) {
+      e.path = jsonPath;
+      e.message = 'Error parsing ' + jsonPath + ': ' + e.message;
+      throw e;
+    }
   }
-}
 
-// Native ERR_REQUIRE_ESM Error is declared here:
-//   https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/errors.js#L1294-L1313
-// Error class factory is implemented here:
-//   function E: https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/errors.js#L323-L341
-//   function makeNodeErrorWithCode: https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/errors.js#L251-L278
-// The code below should create an error that matches the native error as closely as possible.
-// Third-party libraries which attempt to catch the native ERR_REQUIRE_ESM should recognize our imitation error.
-function createErrRequireEsm(filename, parentPath, packageJsonPath) {
-  const code = 'ERR_REQUIRE_ESM'
-  const err = new Error(getMessage(filename, parentPath, packageJsonPath))
-  // Set `name` to be used in stack trace, generate stack trace with that name baked in, then re-declare the `name` field.
-  // This trick is copied from node's source.
-  err.name = `Error [${ code }]`
-  err.stack
-  Object.defineProperty(err, 'name', {
-    value: 'Error',
-    enumerable: false,
-    writable: true,
-    configurable: true
-  })
-  err.code = code
-  return err
+  // Native ERR_REQUIRE_ESM Error is declared here:
+  //   https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/errors.js#L1294-L1313
+  // Error class factory is implemented here:
+  //   function E: https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/errors.js#L323-L341
+  //   function makeNodeErrorWithCode: https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/errors.js#L251-L278
+  // The code below should create an error that matches the native error as closely as possible.
+  // Third-party libraries which attempt to catch the native ERR_REQUIRE_ESM should recognize our imitation error.
+  function createErrRequireEsm(filename, parentPath, packageJsonPath) {
+    const code = 'ERR_REQUIRE_ESM'
+    const err = new Error(getMessage(filename, parentPath, packageJsonPath))
+    // Set `name` to be used in stack trace, generate stack trace with that name baked in, then re-declare the `name` field.
+    // This trick is copied from node's source.
+    err.name = `Error [${ code }]`
+    err.stack
+    Object.defineProperty(err, 'name', {
+      value: 'Error',
+      enumerable: false,
+      writable: true,
+      configurable: true
+    })
+    err.code = code
+    return err
 
-  // Copy-pasted from https://github.com/nodejs/node/blob/b533fb3508009e5f567cc776daba8fbf665386a6/lib/internal/errors.js#L1293-L1311
-  // so that our error message is identical to the native message.
-  function getMessage(filename, parentPath = null, packageJsonPath = null) {
-    const ext = path.extname(filename)
-    let msg = `Must use import to load ES Module: ${filename}`;
-    if (parentPath && packageJsonPath) {
-      const path = require('path');
-      const basename = path.basename(filename) === path.basename(parentPath) ?
-        filename : path.basename(filename);
-      msg +=
-        '\nrequire() of ES modules is not supported.\nrequire() of ' +
-        `${filename} ${parentPath ? `from ${parentPath} ` : ''}` +
-        `is an ES module file as it is a ${ext} file whose nearest parent ` +
-        `package.json contains "type": "module" which defines all ${ext} ` +
-        'files in that package scope as ES modules.\nInstead ' +
-        'change the requiring code to use ' +
-        'import(), or remove "type": "module" from ' +
-        `${packageJsonPath}.\n`;
+    // Copy-pasted from https://github.com/nodejs/node/blob/b533fb3508009e5f567cc776daba8fbf665386a6/lib/internal/errors.js#L1293-L1311
+    // so that our error message is identical to the native message.
+    function getMessage(filename, parentPath = null, packageJsonPath = null) {
+      const ext = path.extname(filename)
+      let msg = `Must use import to load ES Module: ${filename}`;
+      if (parentPath && packageJsonPath) {
+        const path = require('path');
+        const basename = path.basename(filename) === path.basename(parentPath) ?
+          filename : path.basename(filename);
+        msg +=
+          '\nrequire() of ES modules is not supported.\nrequire() of ' +
+          `${filename} ${parentPath ? `from ${parentPath} ` : ''}` +
+          `is an ES module file as it is a ${ext} file whose nearest parent ` +
+          `package.json contains "type": "module" which defines all ${ext} ` +
+          'files in that package scope as ES modules.\nInstead ' +
+          'change the requiring code to use ' +
+          'import(), or remove "type": "module" from ' +
+          `${packageJsonPath}.\n`;
+        return msg;
+      }
       return msg;
     }
-    return msg;
   }
+
+  return {assertScriptCanLoadAsCJSImpl};
 }
+
+module.exports.createNodeCjsLoaderUtils = createNodeCjsLoaderUtils;

dist-raw/node-esm-default-get-format.js

@@ -15,7 +15,6 @@ const experimentalJsonModules = getOptionValue('--experimental-json-modules');
 const experimentalSpeciferResolution =
   getOptionValue('--experimental-specifier-resolution');
 const experimentalWasmModules = getOptionValue('--experimental-wasm-modules');
-const { getPackageType } = require('./node-esm-resolve-implementation.js').createResolve({tsExtensions: [], jsExtensions: []});
 const { URL, fileURLToPath } = require('url');
 const { ERR_UNKNOWN_FILE_EXTENSION } = require('./node-errors').codes;
 
@@ -41,6 +40,9 @@ if (experimentalWasmModules)
 if (experimentalJsonModules)
   extensionFormatMap['.json'] = legacyExtensionFormatMap['.json'] = 'json';
 
+function createDefaultGetFormat(getPackageType) {
+
+// Intentionally unindented to simplify the diff
 function defaultGetFormat(url, context, defaultGetFormatUnused) {
   if (StringPrototypeStartsWith(url, 'node:')) {
     return { format: 'builtin' };
@@ -80,4 +82,9 @@ function defaultGetFormat(url, context, defaultGetFormatUnused) {
   }
   return { format: null };
 }
-exports.defaultGetFormat = defaultGetFormat;
+return {
+  defaultGetFormat: /** @type {import('../src/esm').GetFormatHook} */(defaultGetFormat)
+};
+}
+
+exports.createDefaultGetFormat = createDefaultGetFormat;

dist-raw/node-esm-resolve-implementation.js

@@ -95,16 +95,26 @@ const {
 const CJSModule = Module;
 
 // const packageJsonReader = require('internal/modules/package_json_reader');
-const packageJsonReader = require('./node-package-json-reader');
+
+const { createDefaultGetFormat } = require('./node-esm-default-get-format');
+
 const userConditions = getOptionValue('--conditions');
 const DEFAULT_CONDITIONS = ObjectFreeze(['node', 'import', ...userConditions]);
 const DEFAULT_CONDITIONS_SET = new SafeSet(DEFAULT_CONDITIONS);
 
 const pendingDeprecation = getOptionValue('--pending-deprecation');
 
+/**
+ * @param {{
+ *   tsExtensions: string[];
+ *   jsExtensions: string[];
+ *   preferTsExts: boolean | undefined;
+ *   nodePackageJsonReader: ReturnType<import('./node-package-json-reader').createNodePackageJsonReader>;
+ * }} opts
+ */
 function createResolve(opts) {
-// TODO receive cached fs implementations here
-const {tsExtensions, jsExtensions, preferTsExts} = opts;
+const {tsExtensions, jsExtensions, preferTsExts, nodePackageJsonReader: packageJsonReader} = opts;
+const {defaultGetFormat} = createDefaultGetFormat(getPackageType);
 
 const emittedPackageWarnings = new SafeSet();
 function emitFolderMapDeprecation(match, pjsonUrl, isExports, base) {
@@ -971,9 +981,8 @@ return {
   encodedSepRegEx,
   getPackageType,
   packageExportsResolve,
-  packageImportsResolve
+  packageImportsResolve,
+  defaultGetFormat
 };
 }
-module.exports = {
-  createResolve
-};
+module.exports.createResolve = createResolve;