Added cache for some FS operations while compile

README.md

@@ -543,6 +543,15 @@ Extending `tsconfig.json`:
 
 Note that changes in the extending file while not be respected by `ts-loader`. Its purpose is to satisfy the code editor.
 
+### experimentalFileCaching _(boolean) (default=false)_
+
+By default whenever TypeScript compiler needs to check that file/directory is exist or resolve symlink it makes syscall.
+Also it does not cache the result of this operations and may produce a lot of syscalls with the same arguments ([see comment](https://github.com/TypeStrong/ts-loader/issues/825#issue-354725524) with example).
+In some cases it may produce performance degradation in multiple times.
+
+This flag enables caching for some FS-functions like `fileExists`, `realpath` and `directoryExists` for TypeScript compiler.
+Note that caches are cleared between compilations.
+
 ### `LoaderOptionsPlugin`
 
 [There's a known "gotcha"](https://github.com/TypeStrong/ts-loader/issues/283) if you are using webpack 2 with the `LoaderOptionsPlugin`. If you are faced with the `Cannot read property 'unsafeCache' of undefined` error then you probably need to supply a `resolve` object as below: (Thanks @jeffijoe!)

src/index.ts

@@ -157,7 +157,8 @@ const validLoaderOptions: ValidLoaderOptions[] = [
   'getCustomTransformers',
   'reportFiles',
   'experimentalWatchApi',
-  'allowTsInNodeModules'
+  'allowTsInNodeModules',
+  'experimentalFileCaching'
 ];
 
 /**
@@ -210,7 +211,8 @@ function makeLoaderOptions(instanceName: string, loaderOptions: LoaderOptions) {
       reportFiles: [],
       // When the watch API usage stabilises look to remove this option and make watch usage the default behaviour when available
       experimentalWatchApi: false,
-      allowTsInNodeModules: false
+      allowTsInNodeModules: false,
+      experimentalFileCaching: false
     } as Partial<LoaderOptions>,
     loaderOptions
   );

src/instances.ts

@@ -247,6 +247,12 @@ function successfulTypeScriptInstance(
     colors
   });
 
+  if (!loader._compiler.hooks) {
+    throw new Error(
+      "You may be using an old version of webpack; please check you're using at least version 4"
+    );
+  }
+
   if (loaderOptions.experimentalWatchApi && compiler.createWatchProgram) {
     log.logInfo('Using watch api');
 
@@ -266,17 +272,25 @@ function successfulTypeScriptInstance(
       .getProgram()
       .getProgram();
   } else {
-    const servicesHost = makeServicesHost(scriptRegex, log, loader, instance);
+    const cachedServicesHost = makeServicesHost(
+      scriptRegex,
+      log,
+      loader,
+      instance,
+      loaderOptions.experimentalFileCaching
+    );
+
     instance.languageService = compiler.createLanguageService(
-      servicesHost,
+      cachedServicesHost.servicesHost,
       compiler.createDocumentRegistry()
     );
-  }
 
-  if (!loader._compiler.hooks) {
-    throw new Error(
-      "You may be using an old version of webpack; please check you're using at least version 4"
-    );
+    if (cachedServicesHost.clearCache !== null) {
+      loader._compiler.hooks.watchRun.tap(
+        'ts-loader',
+        cachedServicesHost.clearCache
+      );
+    }
   }
 
   loader._compiler.hooks.afterCompile.tapAsync(

src/interfaces.ts

@@ -302,6 +302,7 @@ export interface LoaderOptions {
     | (() => typescript.CustomTransformers | undefined);
   experimentalWatchApi: boolean;
   allowTsInNodeModules: boolean;
+  experimentalFileCaching: boolean;
 }
 
 export interface TSFile {

src/servicesHost.ts

@@ -14,15 +14,21 @@ import {
   Webpack
 } from './interfaces';
 
+export interface CachedServicesHost {
+  servicesHost: typescript.LanguageServiceHost;
+  clearCache: (() => void) | null;
+}
+
 /**
  * Create the TypeScript language service
  */
 export function makeServicesHost(
   scriptRegex: RegExp,
   log: logger.Logger,
   loader: Webpack,
-  instance: TSInstance
-) {
+  instance: TSInstance,
+  enableFileCaching: boolean
+): CachedServicesHost {
   const {
     compiler,
     compilerOptions,
@@ -52,9 +58,12 @@ export function makeServicesHost(
   const moduleResolutionHost: ModuleResolutionHost = {
     fileExists,
     readFile: readFileWithFallback,
-    realpath: compiler.sys.realpath
+    realpath: compiler.sys.realpath,
+    directoryExists: compiler.sys.directoryExists
   };
 
+  const clearCache = enableFileCaching ? addCache(moduleResolutionHost) : null;
+
   // loader.context seems to work fine on Linux / Mac regardless causes problems for @types resolution on Windows for TypeScript < 2.3
   const getCurrentDirectory = () => loader.context;
 
@@ -97,13 +106,15 @@ export function makeServicesHost(
     /**
      * For @types expansion, these two functions are needed.
      */
-    directoryExists: compiler.sys.directoryExists,
+    directoryExists: moduleResolutionHost.directoryExists,
 
     useCaseSensitiveFileNames: () => compiler.sys.useCaseSensitiveFileNames,
 
+    realpath: moduleResolutionHost.realpath,
+
     // The following three methods are necessary for @types resolution from TS 2.4.1 onwards see: https://github.com/Microsoft/TypeScript/issues/16772
-    fileExists: compiler.sys.fileExists,
-    readFile: compiler.sys.readFile,
+    fileExists: moduleResolutionHost.fileExists,
+    readFile: moduleResolutionHost.readFile,
     readDirectory: compiler.sys.readDirectory,
 
     getCurrentDirectory,
@@ -137,7 +148,7 @@ export function makeServicesHost(
     getCustomTransformers: () => instance.transformers
   };
 
-  return servicesHost;
+  return { servicesHost, clearCache };
 }
 
 /**
@@ -509,3 +520,45 @@ function populateDependencyGraphs(
     ] = true;
   });
 }
+
+function addCache(servicesHost: typescript.ModuleResolutionHost) {
+  const clearCacheFuncs: (() => void)[] = [];
+  if (servicesHost.fileExists !== undefined) {
+    const cache = createCache(servicesHost.fileExists);
+    servicesHost.fileExists = cache.cached;
+    clearCacheFuncs.push(cache.clear);
+  }
+
+  if (servicesHost.directoryExists !== undefined) {
+    const cache = createCache(servicesHost.directoryExists);
+    servicesHost.directoryExists = cache.cached;
+    clearCacheFuncs.push(cache.clear);
+  }
+
+  if (servicesHost.realpath !== undefined) {
+    const cache = createCache(servicesHost.realpath);
+    servicesHost.realpath = cache.cached;
+    clearCacheFuncs.push(cache.clear);
+  }
+
+  return () => clearCacheFuncs.forEach(clear => clear());
+}
+
+function createCache<TOut>(func: (arg: string) => TOut) {
+  let cache: Record<string, TOut | undefined> = {};
+  return {
+    clear: () => {
+      cache = {};
+    },
+    cached: (arg: string) => {
+      let res = cache[arg];
+      if (res !== undefined) {
+        return res;
+      }
+
+      res = func(arg);
+      cache[arg] = res;
+      return res;
+    }
+  };
+}

test/comparison-tests/validateLoaderOptionNames/expectedOutput-2.8/bundle.js

@@ -78,7 +78,7 @@
 /*! no static exports found */
 /***/ (function(module, exports) {
 
-eval("throw new Error(\"Module build failed: Error: ts-loader was supplied with an unexpected loader option: notRealOption/n/nPlease take a look at the options you are supplying; the following are valid options:/nsilent / logLevel / logInfoToStdOut / instance / compiler / context / configFile / transpileOnly / ignoreDiagnostics / errorFormatter / colors / compilerOptions / appendTsSuffixTo / appendTsxSuffixTo / onlyCompileBundledFiles / happyPackMode / getCustomTransformers / reportFiles / experimentalWatchApi / allowTsInNodeModules/n/n    at validateLoaderOptions (/Users/alawson/dev/ts-loader/dist/index.js:110:19)/n    at getLoaderOptions (/Users/alawson/dev/ts-loader/dist/index.js:72:5)/n    at Object.loader (/Users/alawson/dev/ts-loader/dist/index.js:15:21)\");\n\n//# sourceURL=webpack:///./app.ts?");
+eval("throw new Error(\"Module build failed: Error: ts-loader was supplied with an unexpected loader option: notRealOption/n/nPlease take a look at the options you are supplying; the following are valid options:/nsilent / logLevel / logInfoToStdOut / instance / compiler / context / configFile / transpileOnly / ignoreDiagnostics / errorFormatter / colors / compilerOptions / appendTsSuffixTo / appendTsxSuffixTo / onlyCompileBundledFiles / happyPackMode / getCustomTransformers / reportFiles / experimentalWatchApi / allowTsInNodeModules / experimentalFileCaching/n/n    at validateLoaderOptions (/Users/alawson/dev/ts-loader/dist/index.js:110:19)/n    at getLoaderOptions (/Users/alawson/dev/ts-loader/dist/index.js:72:5)/n    at Object.loader (/Users/alawson/dev/ts-loader/dist/index.js:15:21)\");\n\n//# sourceURL=webpack:///./app.ts?");
 
 /***/ })
 

test/comparison-tests/validateLoaderOptionNames/expectedOutput-2.8/output.txt

@@ -7,7 +7,7 @@ ERROR in ./app.ts
 Module build failed: Error: ts-loader was supplied with an unexpected loader option: notRealOption
 
 Please take a look at the options you are supplying; the following are valid options:
-silent / logLevel / logInfoToStdOut / instance / compiler / context / configFile / transpileOnly / ignoreDiagnostics / errorFormatter / colors / compilerOptions / appendTsSuffixTo / appendTsxSuffixTo / onlyCompileBundledFiles / happyPackMode / getCustomTransformers / reportFiles / experimentalWatchApi / allowTsInNodeModules
+silent / logLevel / logInfoToStdOut / instance / compiler / context / configFile / transpileOnly / ignoreDiagnostics / errorFormatter / colors / compilerOptions / appendTsSuffixTo / appendTsxSuffixTo / onlyCompileBundledFiles / happyPackMode / getCustomTransformers / reportFiles / experimentalWatchApi / allowTsInNodeModules / experimentalFileCaching
 
     at validateLoaderOptions (dist/index.js:110:19)
     at getLoaderOptions (dist/index.js:72:5)

test/comparison-tests/validateLoaderOptionNames/expectedOutput-2.9/bundle.js

@@ -93,7 +93,7 @@
 /*! no static exports found */
 /***/ (function(module, exports) {
 
-eval("throw new Error(\"Module build failed: Error: ts-loader was supplied with an unexpected loader option: notRealOption/n/nPlease take a look at the options you are supplying; the following are valid options:/nsilent / logLevel / logInfoToStdOut / instance / compiler / context / configFile / transpileOnly / ignoreDiagnostics / errorFormatter / colors / compilerOptions / appendTsSuffixTo / appendTsxSuffixTo / onlyCompileBundledFiles / happyPackMode / getCustomTransformers / reportFiles / experimentalWatchApi / allowTsInNodeModules/n/n    at validateLoaderOptions (C://source//ts-loader//dist//index.js:110:19)/n    at getLoaderOptions (C://source//ts-loader//dist//index.js:72:5)/n    at Object.loader (C://source//ts-loader//dist//index.js:15:21)\");\n\n//# sourceURL=webpack:///./app.ts?");
+eval("throw new Error(\"Module build failed: Error: ts-loader was supplied with an unexpected loader option: notRealOption/n/nPlease take a look at the options you are supplying; the following are valid options:/nsilent / logLevel / logInfoToStdOut / instance / compiler / context / configFile / transpileOnly / ignoreDiagnostics / errorFormatter / colors / compilerOptions / appendTsSuffixTo / appendTsxSuffixTo / onlyCompileBundledFiles / happyPackMode / getCustomTransformers / reportFiles / experimentalWatchApi / allowTsInNodeModules / experimentalFileCaching/n/n    at validateLoaderOptions (C://source//ts-loader//dist//index.js:110:19)/n    at getLoaderOptions (C://source//ts-loader//dist//index.js:72:5)/n    at Object.loader (C://source//ts-loader//dist//index.js:15:21)\");\n\n//# sourceURL=webpack:///./app.ts?");
 
 /***/ })
 

test/comparison-tests/validateLoaderOptionNames/expectedOutput-2.9/output.txt

@@ -7,7 +7,7 @@ ERROR in ./app.ts
 Module build failed: Error: ts-loader was supplied with an unexpected loader option: notRealOption
 
 Please take a look at the options you are supplying; the following are valid options:
-silent / logLevel / logInfoToStdOut / instance / compiler / context / configFile / transpileOnly / ignoreDiagnostics / errorFormatter / colors / compilerOptions / appendTsSuffixTo / appendTsxSuffixTo / onlyCompileBundledFiles / happyPackMode / getCustomTransformers / reportFiles / experimentalWatchApi / allowTsInNodeModules
+silent / logLevel / logInfoToStdOut / instance / compiler / context / configFile / transpileOnly / ignoreDiagnostics / errorFormatter / colors / compilerOptions / appendTsSuffixTo / appendTsxSuffixTo / onlyCompileBundledFiles / happyPackMode / getCustomTransformers / reportFiles / experimentalWatchApi / allowTsInNodeModules / experimentalFileCaching
 
     at validateLoaderOptions (dist\index.js:110:19)
     at getLoaderOptions (dist\index.js:72:5)

test/comparison-tests/validateLoaderOptionNames/expectedOutput-3.0/bundle.js

@@ -93,7 +93,7 @@
 /*! no static exports found */
 /***/ (function(module, exports) {
 
-eval("throw new Error(\"Module build failed: Error: ts-loader was supplied with an unexpected loader option: notRealOption/n/nPlease take a look at the options you are supplying; the following are valid options:/nsilent / logLevel / logInfoToStdOut / instance / compiler / context / configFile / transpileOnly / ignoreDiagnostics / errorFormatter / colors / compilerOptions / appendTsSuffixTo / appendTsxSuffixTo / onlyCompileBundledFiles / happyPackMode / getCustomTransformers / reportFiles / experimentalWatchApi / allowTsInNodeModules/n/n    at validateLoaderOptions (C://source//ts-loader//dist//index.js:110:19)/n    at getLoaderOptions (C://source//ts-loader//dist//index.js:72:5)/n    at Object.loader (C://source//ts-loader//dist//index.js:15:21)\");\n\n//# sourceURL=webpack:///./app.ts?");
+eval("throw new Error(\"Module build failed: Error: ts-loader was supplied with an unexpected loader option: notRealOption/n/nPlease take a look at the options you are supplying; the following are valid options:/nsilent / logLevel / logInfoToStdOut / instance / compiler / context / configFile / transpileOnly / ignoreDiagnostics / errorFormatter / colors / compilerOptions / appendTsSuffixTo / appendTsxSuffixTo / onlyCompileBundledFiles / happyPackMode / getCustomTransformers / reportFiles / experimentalWatchApi / allowTsInNodeModules / experimentalFileCaching/n/n    at validateLoaderOptions (C://source//ts-loader//dist//index.js:111:19)/n    at getLoaderOptions (C://source//ts-loader//dist//index.js:72:5)/n    at Object.loader (C://source//ts-loader//dist//index.js:15:21)\");\n\n//# sourceURL=webpack:///./app.ts?");
 
 /***/ })
 

test/comparison-tests/validateLoaderOptionNames/expectedOutput-3.0/output.txt

@@ -7,8 +7,8 @@ ERROR in ./app.ts
 Module build failed: Error: ts-loader was supplied with an unexpected loader option: notRealOption
 
 Please take a look at the options you are supplying; the following are valid options:
-silent / logLevel / logInfoToStdOut / instance / compiler / context / configFile / transpileOnly / ignoreDiagnostics / errorFormatter / colors / compilerOptions / appendTsSuffixTo / appendTsxSuffixTo / onlyCompileBundledFiles / happyPackMode / getCustomTransformers / reportFiles / experimentalWatchApi / allowTsInNodeModules
+silent / logLevel / logInfoToStdOut / instance / compiler / context / configFile / transpileOnly / ignoreDiagnostics / errorFormatter / colors / compilerOptions / appendTsSuffixTo / appendTsxSuffixTo / onlyCompileBundledFiles / happyPackMode / getCustomTransformers / reportFiles / experimentalWatchApi / allowTsInNodeModules / experimentalFileCaching
 
-    at validateLoaderOptions (dist\index.js:110:19)
+    at validateLoaderOptions (dist\index.js:111:19)
     at getLoaderOptions (dist\index.js:72:5)
     at Object.loader (dist\index.js:15:21)