feat(jest-runtime): share cacheFS between runtime and transformer

CHANGELOG.md

@@ -10,6 +10,7 @@
 - `[jest-snapshot]` [**BREAKING**] Make prettier optional for inline snapshots - fall back to string replacement ([#7792](https://github.com/facebook/jest/pull/7792))
 - `[jest-runner]` [**BREAKING**] Run transforms over `runnner` ([#8823](https://github.com/facebook/jest/pull/8823))
 - `[jest-runner]` [**BREAKING**] Run transforms over `testRunnner` ([#8823](https://github.com/facebook/jest/pull/8823))
+- `[jest-runtime, jest-transform]` share `cacheFS` between runtime and transformer ([#10901](https://github.com/facebook/jest/pull/10901))
 
 ### Fixes
 

docs/CodeTransformation.md

@@ -30,7 +30,7 @@ interface Transformer<OptionType = unknown> {
    * If V8 coverage is _not_ active, and this is `false`. Jest will instrument the code returned by this transformer using Babel.
    */
   canInstrument?: boolean;
-  createTransformer?: (options?: OptionType) => Transformer;
+  createTransformer?: (options: OptionType) => Transformer;
 
   getCacheKey?: (
     sourceText: string,
@@ -46,6 +46,12 @@ interface Transformer<OptionType = unknown> {
 }
 
 interface TransformOptions {
+  /**
+   * If a transformer does module resolution and reads files, it should populate `cacheFS` so that
+   * Jest avoids reading the same files again, improving performance. `cacheFS` stores entries of
+   * <file path, file contents>
+   */
+  cacheFS: Map<string, string>;
   config: Config.ProjectConfig;
   /** A stringified version of the configuration - useful in cache busting */
   configString: string;

packages/jest-repl/src/cli/repl.ts

@@ -29,6 +29,7 @@ const evalCommand: repl.REPLEval = (
         cmd,
         jestGlobalConfig.replname || 'jest.js',
         {
+          cacheFS: new Map<string, string>(),
           config: jestProjectConfig,
           configString: JSON.stringify(jestProjectConfig),
           instrument: false,

packages/jest-runtime/src/index.ts

@@ -154,7 +154,7 @@ const supportsTopLevelAwait =
   })();
 
 export default class Runtime {
-  private _cacheFS: StringMap;
+  private readonly _cacheFS: StringMap;
   private _config: Config.ProjectConfig;
   private _coverageOptions: ShouldInstrumentOptions;
   private _currentlyExecutingModulePath: string;
@@ -230,7 +230,7 @@ export default class Runtime {
     this._esmoduleRegistry = new Map();
     this._testPath = testPath;
     this._resolver = resolver;
-    this._scriptTransformer = new ScriptTransformer(config);
+    this._scriptTransformer = new ScriptTransformer(config, this._cacheFS);
     this._shouldAutoMock = config.automock;
     this._sourceMapRegistry = new Map();
     this._fileTransforms = new Map();

packages/jest-transform/src/ScriptTransformer.ts

@@ -29,6 +29,7 @@ import shouldInstrument from './shouldInstrument';
 import type {
   Options,
   ReducedTransformOptions,
+  StringMap,
   TransformResult,
   TransformedSource,
   Transformer,
@@ -64,12 +65,17 @@ async function waitForPromiseWithCleanup(
 
 export default class ScriptTransformer {
   private _cache: ProjectCache;
-  private _config: Config.ProjectConfig;
+  private readonly _cacheFS: StringMap;
+  private readonly _config: Config.ProjectConfig;
   private _transformCache: Map<Config.Path, Transformer>;
   private _transformConfigCache: Map<Config.Path, unknown>;
 
-  constructor(config: Config.ProjectConfig) {
+  constructor(
+    config: Config.ProjectConfig,
+    cacheFS: StringMap = new Map<string, string>(),
+  ) {
     this._config = config;
+    this._cacheFS = cacheFS;
     this._transformCache = new Map();
     this._transformConfigCache = new Map();
 
@@ -103,6 +109,7 @@ export default class ScriptTransformer {
         .update(
           transformer.getCacheKey(fileData, filename, {
             ...options,
+            cacheFS: this._cacheFS,
             config: this._config,
             configString,
           }),
@@ -288,6 +295,7 @@ export default class ScriptTransformer {
     if (transform && shouldCallTransform) {
       const processed = transform.process(content, filename, {
         ...options,
+        cacheFS: this._cacheFS,
         config: this._config,
         configString: this._cache.configString,
       });
@@ -375,9 +383,12 @@ export default class ScriptTransformer {
     fileSource?: string,
   ): TransformResult {
     const {isCoreModule, isInternalModule} = options;
-    const content = stripShebang(
-      fileSource || fs.readFileSync(filename, 'utf8'),
-    );
+    let fileContent = fileSource ?? this._cacheFS.get(filename);
+    if (!fileContent) {
+      fileContent = fs.readFileSync(filename, 'utf8');
+      this._cacheFS.set(filename, fileContent);
+    }
+    const content = stripShebang(fileContent);
 
     let code = content;
     let sourceMapPath: string | null = null;

packages/jest-transform/src/__tests__/__snapshots__/script_transformer.test.ts.snap

@@ -7,6 +7,9 @@ exports[`ScriptTransformer passes expected transform options to getCacheKey 1`]
       "module.exports = \\"banana\\";",
       "/fruits/banana.js",
       Object {
+        "cacheFS": Map {
+          "/fruits/banana.js" => "module.exports = \\"banana\\";",
+        },
         "collectCoverage": true,
         "collectCoverageFrom": Array [],
         "collectCoverageOnlyFrom": undefined,
@@ -253,7 +256,7 @@ exports[`ScriptTransformer uses multiple preprocessors 1`] = `
 const TRANSFORMED = {
   filename: '/fruits/banana.js',
   script: 'module.exports = "banana";',
-  config: '{"collectCoverage":false,"collectCoverageFrom":[],"coverageProvider":"babel","supportsDynamicImport":false,"supportsExportNamespaceFrom":false,"supportsStaticESM":false,"supportsTopLevelAwait":false,"instrument":false,"config":{"automock":false,"cache":true,"cacheDirectory":"/cache/","clearMocks":false,"coveragePathIgnorePatterns":[],"cwd":"/test_root_dir/","detectLeaks":false,"detectOpenHandles":false,"errorOnDeprecated":false,"extensionsToTreatAsEsm":[],"extraGlobals":[],"forceCoverageMatch":[],"globals":{},"haste":{},"injectGlobals":true,"moduleDirectories":[],"moduleFileExtensions":["js"],"moduleLoader":"/test_module_loader_path","moduleNameMapper":[],"modulePathIgnorePatterns":[],"modulePaths":[],"name":"test","prettierPath":"prettier","resetMocks":false,"resetModules":false,"restoreMocks":false,"rootDir":"/","roots":[],"runner":"jest-runner","setupFiles":[],"setupFilesAfterEnv":[],"skipFilter":false,"skipNodeResolution":false,"slowTestThreshold":5,"snapshotSerializers":[],"testEnvironment":"node","testEnvironmentOptions":{},"testLocationInResults":false,"testMatch":[],"testPathIgnorePatterns":[],"testRegex":["\\\\.test\\\\.js$"],"testRunner":"jest-circus/runner","testURL":"http://localhost","timers":"real","transform":[["\\\\.js$","test_preprocessor",{}],["\\\\.css$","css-preprocessor",{}]],"transformIgnorePatterns":["/node_modules/"],"watchPathIgnorePatterns":[]},"configString":"{\\"automock\\":false,\\"cache\\":true,\\"cacheDirectory\\":\\"/cache/\\",\\"clearMocks\\":false,\\"coveragePathIgnorePatterns\\":[],\\"cwd\\":\\"/test_root_dir/\\",\\"detectLeaks\\":false,\\"detectOpenHandles\\":false,\\"errorOnDeprecated\\":false,\\"extensionsToTreatAsEsm\\":[],\\"extraGlobals\\":[],\\"forceCoverageMatch\\":[],\\"globals\\":{},\\"haste\\":{},\\"injectGlobals\\":true,\\"moduleDirectories\\":[],\\"moduleFileExtensions\\":[\\"js\\"],\\"moduleLoader\\":\\"/test_module_loader_path\\",\\"moduleNameMapper\\":[],\\"modulePathIgnorePatterns\\":[],\\"modulePaths\\":[],\\"name\\":\\"test\\",\\"prettierPath\\":\\"prettier\\",\\"resetMocks\\":false,\\"resetModules\\":false,\\"restoreMocks\\":false,\\"rootDir\\":\\"/\\",\\"roots\\":[],\\"runner\\":\\"jest-runner\\",\\"setupFiles\\":[],\\"setupFilesAfterEnv\\":[],\\"skipFilter\\":false,\\"skipNodeResolution\\":false,\\"slowTestThreshold\\":5,\\"snapshotSerializers\\":[],\\"testEnvironment\\":\\"node\\",\\"testEnvironmentOptions\\":{},\\"testLocationInResults\\":false,\\"testMatch\\":[],\\"testPathIgnorePatterns\\":[],\\"testRegex\\":[\\"\\\\\\\\.test\\\\\\\\.js$\\"],\\"testRunner\\":\\"jest-circus/runner\\",\\"testURL\\":\\"http://localhost\\",\\"timers\\":\\"real\\",\\"transform\\":[[\\"\\\\\\\\.js$\\",\\"test_preprocessor\\",{}],[\\"\\\\\\\\.css$\\",\\"css-preprocessor\\",{}]],\\"transformIgnorePatterns\\":[\\"/node_modules/\\"],\\"watchPathIgnorePatterns\\":[]}"}',
+  config: '{"collectCoverage":false,"collectCoverageFrom":[],"coverageProvider":"babel","supportsDynamicImport":false,"supportsExportNamespaceFrom":false,"supportsStaticESM":false,"supportsTopLevelAwait":false,"instrument":false,"cacheFS":{},"config":{"automock":false,"cache":true,"cacheDirectory":"/cache/","clearMocks":false,"coveragePathIgnorePatterns":[],"cwd":"/test_root_dir/","detectLeaks":false,"detectOpenHandles":false,"errorOnDeprecated":false,"extensionsToTreatAsEsm":[],"extraGlobals":[],"forceCoverageMatch":[],"globals":{},"haste":{},"injectGlobals":true,"moduleDirectories":[],"moduleFileExtensions":["js"],"moduleLoader":"/test_module_loader_path","moduleNameMapper":[],"modulePathIgnorePatterns":[],"modulePaths":[],"name":"test","prettierPath":"prettier","resetMocks":false,"resetModules":false,"restoreMocks":false,"rootDir":"/","roots":[],"runner":"jest-runner","setupFiles":[],"setupFilesAfterEnv":[],"skipFilter":false,"skipNodeResolution":false,"slowTestThreshold":5,"snapshotSerializers":[],"testEnvironment":"node","testEnvironmentOptions":{},"testLocationInResults":false,"testMatch":[],"testPathIgnorePatterns":[],"testRegex":["\\\\.test\\\\.js$"],"testRunner":"jest-circus/runner","testURL":"http://localhost","timers":"real","transform":[["\\\\.js$","test_preprocessor",{}],["\\\\.css$","css-preprocessor",{}]],"transformIgnorePatterns":["/node_modules/"],"watchPathIgnorePatterns":[]},"configString":"{\\"automock\\":false,\\"cache\\":true,\\"cacheDirectory\\":\\"/cache/\\",\\"clearMocks\\":false,\\"coveragePathIgnorePatterns\\":[],\\"cwd\\":\\"/test_root_dir/\\",\\"detectLeaks\\":false,\\"detectOpenHandles\\":false,\\"errorOnDeprecated\\":false,\\"extensionsToTreatAsEsm\\":[],\\"extraGlobals\\":[],\\"forceCoverageMatch\\":[],\\"globals\\":{},\\"haste\\":{},\\"injectGlobals\\":true,\\"moduleDirectories\\":[],\\"moduleFileExtensions\\":[\\"js\\"],\\"moduleLoader\\":\\"/test_module_loader_path\\",\\"moduleNameMapper\\":[],\\"modulePathIgnorePatterns\\":[],\\"modulePaths\\":[],\\"name\\":\\"test\\",\\"prettierPath\\":\\"prettier\\",\\"resetMocks\\":false,\\"resetModules\\":false,\\"restoreMocks\\":false,\\"rootDir\\":\\"/\\",\\"roots\\":[],\\"runner\\":\\"jest-runner\\",\\"setupFiles\\":[],\\"setupFilesAfterEnv\\":[],\\"skipFilter\\":false,\\"skipNodeResolution\\":false,\\"slowTestThreshold\\":5,\\"snapshotSerializers\\":[],\\"testEnvironment\\":\\"node\\",\\"testEnvironmentOptions\\":{},\\"testLocationInResults\\":false,\\"testMatch\\":[],\\"testPathIgnorePatterns\\":[],\\"testRegex\\":[\\"\\\\\\\\.test\\\\\\\\.js$\\"],\\"testRunner\\":\\"jest-circus/runner\\",\\"testURL\\":\\"http://localhost\\",\\"timers\\":\\"real\\",\\"transform\\":[[\\"\\\\\\\\.js$\\",\\"test_preprocessor\\",{}],[\\"\\\\\\\\.css$\\",\\"css-preprocessor\\",{}]],\\"transformIgnorePatterns\\":[\\"/node_modules/\\"],\\"watchPathIgnorePatterns\\":[]}"}',
 };
 `;
 
@@ -270,7 +273,7 @@ exports[`ScriptTransformer uses the supplied preprocessor 1`] = `
 const TRANSFORMED = {
   filename: '/fruits/banana.js',
   script: 'module.exports = "banana";',
-  config: '{"collectCoverage":false,"collectCoverageFrom":[],"coverageProvider":"babel","supportsDynamicImport":false,"supportsExportNamespaceFrom":false,"supportsStaticESM":false,"supportsTopLevelAwait":false,"instrument":false,"config":{"automock":false,"cache":true,"cacheDirectory":"/cache/","clearMocks":false,"coveragePathIgnorePatterns":[],"cwd":"/test_root_dir/","detectLeaks":false,"detectOpenHandles":false,"errorOnDeprecated":false,"extensionsToTreatAsEsm":[],"extraGlobals":[],"forceCoverageMatch":[],"globals":{},"haste":{},"injectGlobals":true,"moduleDirectories":[],"moduleFileExtensions":["js"],"moduleLoader":"/test_module_loader_path","moduleNameMapper":[],"modulePathIgnorePatterns":[],"modulePaths":[],"name":"test","prettierPath":"prettier","resetMocks":false,"resetModules":false,"restoreMocks":false,"rootDir":"/","roots":[],"runner":"jest-runner","setupFiles":[],"setupFilesAfterEnv":[],"skipFilter":false,"skipNodeResolution":false,"slowTestThreshold":5,"snapshotSerializers":[],"testEnvironment":"node","testEnvironmentOptions":{},"testLocationInResults":false,"testMatch":[],"testPathIgnorePatterns":[],"testRegex":["\\\\.test\\\\.js$"],"testRunner":"jest-circus/runner","testURL":"http://localhost","timers":"real","transform":[["\\\\.js$","test_preprocessor",{}]],"transformIgnorePatterns":["/node_modules/"],"watchPathIgnorePatterns":[]},"configString":"{\\"automock\\":false,\\"cache\\":true,\\"cacheDirectory\\":\\"/cache/\\",\\"clearMocks\\":false,\\"coveragePathIgnorePatterns\\":[],\\"cwd\\":\\"/test_root_dir/\\",\\"detectLeaks\\":false,\\"detectOpenHandles\\":false,\\"errorOnDeprecated\\":false,\\"extensionsToTreatAsEsm\\":[],\\"extraGlobals\\":[],\\"forceCoverageMatch\\":[],\\"globals\\":{},\\"haste\\":{},\\"injectGlobals\\":true,\\"moduleDirectories\\":[],\\"moduleFileExtensions\\":[\\"js\\"],\\"moduleLoader\\":\\"/test_module_loader_path\\",\\"moduleNameMapper\\":[],\\"modulePathIgnorePatterns\\":[],\\"modulePaths\\":[],\\"name\\":\\"test\\",\\"prettierPath\\":\\"prettier\\",\\"resetMocks\\":false,\\"resetModules\\":false,\\"restoreMocks\\":false,\\"rootDir\\":\\"/\\",\\"roots\\":[],\\"runner\\":\\"jest-runner\\",\\"setupFiles\\":[],\\"setupFilesAfterEnv\\":[],\\"skipFilter\\":false,\\"skipNodeResolution\\":false,\\"slowTestThreshold\\":5,\\"snapshotSerializers\\":[],\\"testEnvironment\\":\\"node\\",\\"testEnvironmentOptions\\":{},\\"testLocationInResults\\":false,\\"testMatch\\":[],\\"testPathIgnorePatterns\\":[],\\"testRegex\\":[\\"\\\\\\\\.test\\\\\\\\.js$\\"],\\"testRunner\\":\\"jest-circus/runner\\",\\"testURL\\":\\"http://localhost\\",\\"timers\\":\\"real\\",\\"transform\\":[[\\"\\\\\\\\.js$\\",\\"test_preprocessor\\",{}]],\\"transformIgnorePatterns\\":[\\"/node_modules/\\"],\\"watchPathIgnorePatterns\\":[]}"}',
+  config: '{"collectCoverage":false,"collectCoverageFrom":[],"coverageProvider":"babel","supportsDynamicImport":false,"supportsExportNamespaceFrom":false,"supportsStaticESM":false,"supportsTopLevelAwait":false,"instrument":false,"cacheFS":{},"config":{"automock":false,"cache":true,"cacheDirectory":"/cache/","clearMocks":false,"coveragePathIgnorePatterns":[],"cwd":"/test_root_dir/","detectLeaks":false,"detectOpenHandles":false,"errorOnDeprecated":false,"extensionsToTreatAsEsm":[],"extraGlobals":[],"forceCoverageMatch":[],"globals":{},"haste":{},"injectGlobals":true,"moduleDirectories":[],"moduleFileExtensions":["js"],"moduleLoader":"/test_module_loader_path","moduleNameMapper":[],"modulePathIgnorePatterns":[],"modulePaths":[],"name":"test","prettierPath":"prettier","resetMocks":false,"resetModules":false,"restoreMocks":false,"rootDir":"/","roots":[],"runner":"jest-runner","setupFiles":[],"setupFilesAfterEnv":[],"skipFilter":false,"skipNodeResolution":false,"slowTestThreshold":5,"snapshotSerializers":[],"testEnvironment":"node","testEnvironmentOptions":{},"testLocationInResults":false,"testMatch":[],"testPathIgnorePatterns":[],"testRegex":["\\\\.test\\\\.js$"],"testRunner":"jest-circus/runner","testURL":"http://localhost","timers":"real","transform":[["\\\\.js$","test_preprocessor",{}]],"transformIgnorePatterns":["/node_modules/"],"watchPathIgnorePatterns":[]},"configString":"{\\"automock\\":false,\\"cache\\":true,\\"cacheDirectory\\":\\"/cache/\\",\\"clearMocks\\":false,\\"coveragePathIgnorePatterns\\":[],\\"cwd\\":\\"/test_root_dir/\\",\\"detectLeaks\\":false,\\"detectOpenHandles\\":false,\\"errorOnDeprecated\\":false,\\"extensionsToTreatAsEsm\\":[],\\"extraGlobals\\":[],\\"forceCoverageMatch\\":[],\\"globals\\":{},\\"haste\\":{},\\"injectGlobals\\":true,\\"moduleDirectories\\":[],\\"moduleFileExtensions\\":[\\"js\\"],\\"moduleLoader\\":\\"/test_module_loader_path\\",\\"moduleNameMapper\\":[],\\"modulePathIgnorePatterns\\":[],\\"modulePaths\\":[],\\"name\\":\\"test\\",\\"prettierPath\\":\\"prettier\\",\\"resetMocks\\":false,\\"resetModules\\":false,\\"restoreMocks\\":false,\\"rootDir\\":\\"/\\",\\"roots\\":[],\\"runner\\":\\"jest-runner\\",\\"setupFiles\\":[],\\"setupFilesAfterEnv\\":[],\\"skipFilter\\":false,\\"skipNodeResolution\\":false,\\"slowTestThreshold\\":5,\\"snapshotSerializers\\":[],\\"testEnvironment\\":\\"node\\",\\"testEnvironmentOptions\\":{},\\"testLocationInResults\\":false,\\"testMatch\\":[],\\"testPathIgnorePatterns\\":[],\\"testRegex\\":[\\"\\\\\\\\.test\\\\\\\\.js$\\"],\\"testRunner\\":\\"jest-circus/runner\\",\\"testURL\\":\\"http://localhost\\",\\"timers\\":\\"real\\",\\"transform\\":[[\\"\\\\\\\\.js$\\",\\"test_preprocessor\\",{}]],\\"transformIgnorePatterns\\":[\\"/node_modules/\\"],\\"watchPathIgnorePatterns\\":[]}"}',
 };
 `;
 

packages/jest-transform/src/__tests__/script_transformer.test.ts

@@ -77,6 +77,15 @@ jest.mock(
   {virtual: true},
 );
 
+jest.mock(
+  'cache_fs_preprocessor',
+  () => ({
+    getCacheKey: jest.fn(() => 'ab'),
+    process: jest.fn(() => 'processedCode'),
+  }),
+  {virtual: true},
+);
+
 jest.mock(
   'preprocessor-with-sourcemaps',
   () => ({
@@ -671,8 +680,8 @@ describe('ScriptTransformer', () => {
     config = Object.assign(config, {
       transform: [['\\.js$', 'configureable-preprocessor', transformerConfig]],
     });
-
     const scriptTransformer = new ScriptTransformer(config);
+
     scriptTransformer.transform('/fruits/banana.js', {});
     expect(
       require('configureable-preprocessor').createTransformer,
@@ -751,6 +760,31 @@ describe('ScriptTransformer', () => {
     expect(writeFileAtomic.sync).not.toBeCalled();
   });
 
+  it('should reuse the value from in-memory cache which is set by custom transformer', () => {
+    const cacheFS = new Map<string, string>();
+    const testPreprocessor = require('cache_fs_preprocessor');
+    const scriptTransformer = new ScriptTransformer(
+      {
+        ...config,
+        transform: [['\\.js$', 'cache_fs_preprocessor', {}]],
+      },
+      cacheFS,
+    );
+    const fileName1 = '/fruits/banana.js';
+    const fileName2 = '/fruits/kiwi.js';
+
+    scriptTransformer.transform(fileName1, getCoverageOptions());
+
+    cacheFS.set(fileName2, 'foo');
+
+    scriptTransformer.transform(fileName2, getCoverageOptions());
+
+    expect(testPreprocessor.getCacheKey.mock.calls[0][2].cacheFS).toBeDefined()
+    expect(testPreprocessor.process.mock.calls[0][2].cacheFS).toBeDefined()
+    expect(fs.readFileSync).toHaveBeenCalledTimes(1);
+    expect(fs.readFileSync).toBeCalledWith(fileName1, 'utf8');
+  });
+
   it('does not reuse the in-memory cache between different projects', () => {
     const scriptTransformer = new ScriptTransformer({
       ...config,

packages/jest-transform/src/types.ts

@@ -50,7 +50,11 @@ export interface ReducedTransformOptions extends CallerTransformOptions {
   instrument: boolean;
 }
 
+export type StringMap = Map<string, string>;
+
 export interface TransformOptions extends ReducedTransformOptions {
+  /** a cached file system which is used in jest-runtime - useful to improve performance */
+  cacheFS: StringMap;
   config: Config.ProjectConfig;
   /** A stringified version of the configuration - useful in cache busting */
   configString: string;