perf(ngcc): performance improvements

packages/compiler-cli/ngcc/src/execution/create_compile_function.ts

@@ -14,6 +14,7 @@ import {Logger} from '../../../src/ngtsc/logging';
 import {ParsedConfiguration} from '../../../src/perform_compile';
 import {getEntryPointFormat} from '../packages/entry_point';
 import {makeEntryPointBundle} from '../packages/entry_point_bundle';
+import {createModuleResolutionCache, SharedFileCache} from '../packages/source_file_cache';
 import {PathMappings} from '../path_mappings';
 import {FileWriter} from '../writing/file_writer';
 
@@ -30,6 +31,8 @@ export function getCreateCompileFn(
   return (beforeWritingFiles, onTaskCompleted) => {
     const {Transformer} = require('../packages/transformer');
     const transformer = new Transformer(fileSystem, logger, tsConfig);
+    const sharedFileCache = new SharedFileCache(fileSystem);
+    const moduleResolutionCache = createModuleResolutionCache(fileSystem);
 
     return (task: Task) => {
       const {entryPoint, formatProperty, formatPropertiesToMarkAsProcessed, processDts} = task;
@@ -54,8 +57,8 @@ export function getCreateCompileFn(
       logger.info(`Compiling ${entryPoint.name} : ${formatProperty} as ${format}`);
 
       const bundle = makeEntryPointBundle(
-          fileSystem, entryPoint, formatPath, isCore, format, processDts, pathMappings, true,
-          enableI18nLegacyMessageIdFormat);
+          fileSystem, entryPoint, sharedFileCache, moduleResolutionCache, formatPath, isCore,
+          format, processDts, pathMappings, true, enableI18nLegacyMessageIdFormat);
 
       const result = transformer.transform(bundle);
       if (result.success) {

packages/compiler-cli/ngcc/src/main.ts

@@ -8,8 +8,6 @@
 
 /// <reference types="node" />
 
-import * as os from 'os';
-
 import {AbsoluteFsPath, FileSystem, resolve} from '../../src/ngtsc/file_system';
 import {Logger} from '../../src/ngtsc/logging';
 import {ParsedConfiguration} from '../../src/perform_compile';
@@ -35,7 +33,7 @@ import {composeTaskCompletedCallbacks, createLogErrorHandler, createMarkAsProces
 import {AsyncLocker} from './locking/async_locker';
 import {LockFileWithChildProcess} from './locking/lock_file_with_child_process';
 import {SyncLocker} from './locking/sync_locker';
-import {AsyncNgccOptions, getSharedSetup, SyncNgccOptions} from './ngcc_options';
+import {AsyncNgccOptions, getMaxNumberOfWorkers, getSharedSetup, SyncNgccOptions} from './ngcc_options';
 import {NgccConfiguration} from './packages/configuration';
 import {EntryPointJsonProperty, SUPPORTED_FORMAT_PROPERTIES} from './packages/entry_point';
 import {EntryPointManifest, InvalidatingEntryPointManifest} from './packages/entry_point_manifest';
@@ -92,10 +90,9 @@ export function mainNgcc(options: AsyncNgccOptions|SyncNgccOptions): void|Promis
     return;
   }
 
-  // Execute in parallel, if async execution is acceptable and there are more than 2 CPU cores.
-  // (One CPU core is always reserved for the master process and we need at least 2 worker processes
-  // in order to run tasks in parallel.)
-  const inParallel = async && (os.cpus().length > 2);
+  // Determine the number of workers to use and whether ngcc should run in parallel.
+  const workerCount = async ? getMaxNumberOfWorkers() : 1;
+  const inParallel = workerCount > 1;
 
   const analyzeEntryPoints = getAnalyzeEntryPointsFn(
       logger, finder, fileSystem, supportedPropertiesToConsider, compileAllFormats,
@@ -113,7 +110,7 @@ export function mainNgcc(options: AsyncNgccOptions|SyncNgccOptions): void|Promis
   const createTaskCompletedCallback =
       getCreateTaskCompletedCallback(pkgJsonUpdater, errorOnFailedEntryPoint, logger, fileSystem);
   const executor = getExecutor(
-      async, inParallel, logger, fileWriter, pkgJsonUpdater, fileSystem, config,
+      async, workerCount, logger, fileWriter, pkgJsonUpdater, fileSystem, config,
       createTaskCompletedCallback);
 
   return executor.execute(analyzeEntryPoints, createCompileFn);
@@ -153,17 +150,16 @@ function getCreateTaskCompletedCallback(
 }
 
 function getExecutor(
-    async: boolean, inParallel: boolean, logger: Logger, fileWriter: FileWriter,
+    async: boolean, workerCount: number, logger: Logger, fileWriter: FileWriter,
     pkgJsonUpdater: PackageJsonUpdater, fileSystem: FileSystem, config: NgccConfiguration,
     createTaskCompletedCallback: CreateTaskCompletedCallback): Executor {
   const lockFile = new LockFileWithChildProcess(fileSystem, logger);
   if (async) {
     // Execute asynchronously (either serially or in parallel)
     const {retryAttempts, retryDelay} = config.getLockingConfig();
     const locker = new AsyncLocker(lockFile, logger, retryDelay, retryAttempts);
-    if (inParallel) {
-      // Execute in parallel. Use up to 8 CPU cores for workers, always reserving one for master.
-      const workerCount = Math.min(8, os.cpus().length - 1);
+    if (workerCount > 1) {
+      // Execute in parallel.
       return new ClusterExecutor(
           workerCount, fileSystem, logger, fileWriter, pkgJsonUpdater, locker,
           createTaskCompletedCallback);

packages/compiler-cli/ngcc/src/ngcc_options.ts

@@ -5,6 +5,8 @@
  * Use of this source code is governed by an MIT-style license that can be
  * found in the LICENSE file at https://angular.io/license
  */
+import * as os from 'os';
+
 import {absoluteFrom, AbsoluteFsPath, FileSystem, getFileSystem} from '../../src/ngtsc/file_system';
 import {ConsoleLogger, Logger, LogLevel} from '../../src/ngtsc/logging';
 import {ParsedConfiguration, readConfiguration} from '../../src/perform_compile';
@@ -254,3 +256,26 @@ function checkForSolutionStyleTsConfig(
         `  ngcc ... --tsconfig "${fileSystem.relative(projectPath, tsConfig.project)}"`);
   }
 }
+
+/**
+ * Determines the maximum number of workers to use for parallel execution. This can be set using the
+ * NGCC_MAX_WORKERS environment variable, or is computed based on the number of available CPUs. One
+ * CPU core is always reserved for the master process, so we take the number of CPUs minus one, with
+ * a maximum of 4 workers. We don't scale the number of workers beyond 4 by default, as it takes
+ * considerably more memory and CPU cycles while not offering a substantial improvement in time.
+ */
+export function getMaxNumberOfWorkers(): number {
+  const maxWorkers = process.env.NGCC_MAX_WORKERS;
+  if (maxWorkers === undefined) {
+    // Use up to 4 CPU cores for workers, always reserving one for master.
+    return Math.max(1, Math.min(4, os.cpus().length - 1));
+  }
+
+  const numericMaxWorkers = +maxWorkers.trim();
+  if (!Number.isInteger(numericMaxWorkers)) {
+    throw new Error('NGCC_MAX_WORKERS should be an integer.');
+  } else if (numericMaxWorkers < 1) {
+    throw new Error('NGCC_MAX_WORKERS should be at least 1.');
+  }
+  return numericMaxWorkers;
+}

packages/compiler-cli/ngcc/src/packages/entry_point_bundle.ts

@@ -6,11 +6,12 @@
  * found in the LICENSE file at https://angular.io/license
  */
 import * as ts from 'typescript';
-import {AbsoluteFsPath, FileSystem, NgtscCompilerHost} from '../../../src/ngtsc/file_system';
+import {AbsoluteFsPath, FileSystem} from '../../../src/ngtsc/file_system';
 import {PathMappings} from '../path_mappings';
 import {BundleProgram, makeBundleProgram} from './bundle_program';
 import {EntryPoint, EntryPointFormat} from './entry_point';
-import {NgccSourcesCompilerHost} from './ngcc_compiler_host';
+import {NgccDtsCompilerHost, NgccSourcesCompilerHost} from './ngcc_compiler_host';
+import {EntryPointFileCache, SharedFileCache} from './source_file_cache';
 
 /**
  * A bundle of files and paths (and TS programs) that correspond to a particular
@@ -31,6 +32,8 @@ export interface EntryPointBundle {
  * Get an object that describes a formatted bundle for an entry-point.
  * @param fs The current file-system being used.
  * @param entryPoint The entry-point that contains the bundle.
+ * @param sharedFileCache The cache to use for source files that are shared across all entry-points.
+ * @param moduleResolutionCache The module resolution cache to use.
  * @param formatPath The path to the source files for this bundle.
  * @param isCore This entry point is the Angular core package.
  * @param format The underlying format of the bundle.
@@ -42,16 +45,19 @@ export interface EntryPointBundle {
  * component templates.
  */
 export function makeEntryPointBundle(
-    fs: FileSystem, entryPoint: EntryPoint, formatPath: string, isCore: boolean,
+    fs: FileSystem, entryPoint: EntryPoint, sharedFileCache: SharedFileCache,
+    moduleResolutionCache: ts.ModuleResolutionCache, formatPath: string, isCore: boolean,
     format: EntryPointFormat, transformDts: boolean, pathMappings?: PathMappings,
     mirrorDtsFromSrc: boolean = false,
     enableI18nLegacyMessageIdFormat: boolean = true): EntryPointBundle {
   // Create the TS program and necessary helpers.
   const rootDir = entryPoint.packagePath;
   const options: ts
       .CompilerOptions = {allowJs: true, maxNodeModuleJsDepth: Infinity, rootDir, ...pathMappings};
-  const srcHost = new NgccSourcesCompilerHost(fs, options, entryPoint.packagePath);
-  const dtsHost = new NgtscCompilerHost(fs, options);
+  const entryPointCache = new EntryPointFileCache(fs, sharedFileCache);
+  const dtsHost = new NgccDtsCompilerHost(fs, options, entryPointCache, moduleResolutionCache);
+  const srcHost = new NgccSourcesCompilerHost(
+      fs, options, entryPointCache, moduleResolutionCache, entryPoint.packagePath);
 
   // Create the bundle programs, as necessary.
   const absFormatPath = fs.resolve(entryPoint.path, formatPath);

packages/compiler-cli/ngcc/src/packages/ngcc_compiler_host.ts

@@ -10,6 +10,7 @@ import * as ts from 'typescript';
 import {AbsoluteFsPath, FileSystem, NgtscCompilerHost} from '../../../src/ngtsc/file_system';
 import {isWithinPackage} from '../analysis/util';
 import {isRelativePath} from '../utils';
+import {EntryPointFileCache} from './source_file_cache';
 
 /**
  * Represents a compiler host that resolves a module import as a JavaScript source file if
@@ -18,19 +19,24 @@ import {isRelativePath} from '../utils';
  * would otherwise let TypeScript prefer the .d.ts file instead of the JavaScript source file.
  */
 export class NgccSourcesCompilerHost extends NgtscCompilerHost {
-  private cache = ts.createModuleResolutionCache(
-      this.getCurrentDirectory(), file => this.getCanonicalFileName(file));
-
-  constructor(fs: FileSystem, options: ts.CompilerOptions, protected packagePath: AbsoluteFsPath) {
+  constructor(
+      fs: FileSystem, options: ts.CompilerOptions, private cache: EntryPointFileCache,
+      private moduleResolutionCache: ts.ModuleResolutionCache,
+      protected packagePath: AbsoluteFsPath) {
     super(fs, options);
   }
 
+  getSourceFile(fileName: string, languageVersion: ts.ScriptTarget): ts.SourceFile|undefined {
+    return this.cache.getCachedSourceFile(fileName, languageVersion);
+  }
+
   resolveModuleNames(
       moduleNames: string[], containingFile: string, reusedNames?: string[],
       redirectedReference?: ts.ResolvedProjectReference): Array<ts.ResolvedModule|undefined> {
     return moduleNames.map(moduleName => {
       const {resolvedModule} = ts.resolveModuleName(
-          moduleName, containingFile, this.options, this, this.cache, redirectedReference);
+          moduleName, containingFile, this.options, this, this.moduleResolutionCache,
+          redirectedReference);
 
       // If the module request originated from a relative import in a JavaScript source file,
       // TypeScript may have resolved the module to its .d.ts declaration file if the .js source
@@ -59,3 +65,31 @@ export class NgccSourcesCompilerHost extends NgtscCompilerHost {
     });
   }
 }
+
+/**
+ * A compiler host implementation that is used for the typings program. It leverages the entry-point
+ * cache for source files and module resolution, as these results can be reused across the sources
+ * program.
+ */
+export class NgccDtsCompilerHost extends NgtscCompilerHost {
+  constructor(
+      fs: FileSystem, options: ts.CompilerOptions, private cache: EntryPointFileCache,
+      private moduleResolutionCache: ts.ModuleResolutionCache) {
+    super(fs, options);
+  }
+
+  getSourceFile(fileName: string, languageVersion: ts.ScriptTarget): ts.SourceFile|undefined {
+    return this.cache.getCachedSourceFile(fileName, languageVersion);
+  }
+
+  resolveModuleNames(
+      moduleNames: string[], containingFile: string, reusedNames?: string[],
+      redirectedReference?: ts.ResolvedProjectReference): Array<ts.ResolvedModule|undefined> {
+    return moduleNames.map(moduleName => {
+      const {resolvedModule} = ts.resolveModuleName(
+          moduleName, containingFile, this.options, this, this.moduleResolutionCache,
+          redirectedReference);
+      return resolvedModule;
+    });
+  }
+}