From a4f405f64d13261edde6b4c4c97d8b0cd5ef5f1e Mon Sep 17 00:00:00 2001 From: JoostK Date: Mon, 14 Sep 2020 14:08:29 +0200 Subject: [PATCH] perf(ngcc): introduce cache for sharing data across entry-points ngcc creates typically two `ts.Program` instances for each entry-point, one for processing sources and another one for processing the typings. The creation of these programs is somewhat expensive, as it concerns module resolution and parsing of source files. This commit implements several layers of caching to optimize the creation of programs: 1. A shared module resolution cache across all entry-points within a single invocation of ngcc. Both the sources and typings program benefit from this cache. 2. Sharing the parsed `ts.SourceFile` for a single entry-point between the sources and typings program. 3. Sharing parsed `ts.SourceFile`s of TypeScript's default libraries across all entry-points within a single invocation. Some of these default library typings are large and therefore expensive to parse, so sharing the parsed source files across all entry-points offers a significant performance improvement. Using a bare CLI app created using `ng new` + `ng add @angular/material`, the above changes offer a ~3x improvement in ngcc's running time. --- .../src/execution/create_compile_function.ts | 6 +- .../ngcc/src/packages/entry_point_bundle.ts | 15 +-- .../ngcc/src/packages/ngcc_compiler_host.ts | 41 +++++++- .../ngcc/src/packages/transform_cache.ts | 96 +++++++++++++++++++ .../compiler-cli/ngcc/test/helpers/utils.ts | 4 +- .../test/packages/entry_point_bundle_spec.ts | 19 ++-- .../new_entry_point_file_writer_spec.ts | 4 +- 7 files changed, 164 insertions(+), 21 deletions(-) create mode 100644 packages/compiler-cli/ngcc/src/packages/transform_cache.ts diff --git a/packages/compiler-cli/ngcc/src/execution/create_compile_function.ts b/packages/compiler-cli/ngcc/src/execution/create_compile_function.ts index c3671e6946a565..69b5752c134788 100644 --- a/packages/compiler-cli/ngcc/src/execution/create_compile_function.ts +++ b/packages/compiler-cli/ngcc/src/execution/create_compile_function.ts @@ -14,6 +14,7 @@ import {Logger} from '../../../src/ngtsc/logging'; import {ParsedConfiguration} from '../../../src/perform_compile'; import {getEntryPointFormat} from '../packages/entry_point'; import {makeEntryPointBundle} from '../packages/entry_point_bundle'; +import {TransformCache} from '../packages/transform_cache'; import {PathMappings} from '../path_mappings'; import {FileWriter} from '../writing/file_writer'; @@ -30,6 +31,7 @@ export function getCreateCompileFn( return (beforeWritingFiles, onTaskCompleted) => { const {Transformer} = require('../packages/transformer'); const transformer = new Transformer(fileSystem, logger, tsConfig); + const compilationCache = new TransformCache(fileSystem); return (task: Task) => { const {entryPoint, formatProperty, formatPropertiesToMarkAsProcessed, processDts} = task; @@ -54,8 +56,8 @@ export function getCreateCompileFn( logger.info(`Compiling ${entryPoint.name} : ${formatProperty} as ${format}`); const bundle = makeEntryPointBundle( - fileSystem, entryPoint, formatPath, isCore, format, processDts, pathMappings, true, - enableI18nLegacyMessageIdFormat); + fileSystem, entryPoint, compilationCache, formatPath, isCore, format, processDts, + pathMappings, true, enableI18nLegacyMessageIdFormat); const result = transformer.transform(bundle); if (result.success) { diff --git a/packages/compiler-cli/ngcc/src/packages/entry_point_bundle.ts b/packages/compiler-cli/ngcc/src/packages/entry_point_bundle.ts index 68eea9495df185..fec8b5b41f241f 100644 --- a/packages/compiler-cli/ngcc/src/packages/entry_point_bundle.ts +++ b/packages/compiler-cli/ngcc/src/packages/entry_point_bundle.ts @@ -6,11 +6,12 @@ * found in the LICENSE file at https://angular.io/license */ import * as ts from 'typescript'; -import {AbsoluteFsPath, FileSystem, NgtscCompilerHost} from '../../../src/ngtsc/file_system'; +import {AbsoluteFsPath, FileSystem} from '../../../src/ngtsc/file_system'; import {PathMappings} from '../path_mappings'; import {BundleProgram, makeBundleProgram} from './bundle_program'; import {EntryPoint, EntryPointFormat} from './entry_point'; -import {NgccSourcesCompilerHost} from './ngcc_compiler_host'; +import {NgccDtsCompilerHost, NgccSourcesCompilerHost} from './ngcc_compiler_host'; +import {EntryPointCache, TransformCache} from './transform_cache'; /** * A bundle of files and paths (and TS programs) that correspond to a particular @@ -31,6 +32,7 @@ export interface EntryPointBundle { * Get an object that describes a formatted bundle for an entry-point. * @param fs The current file-system being used. * @param entryPoint The entry-point that contains the bundle. + * @param transformCache The cache to use for data that is shared across all entry-points. * @param formatPath The path to the source files for this bundle. * @param isCore This entry point is the Angular core package. * @param format The underlying format of the bundle. @@ -42,16 +44,17 @@ export interface EntryPointBundle { * component templates. */ export function makeEntryPointBundle( - fs: FileSystem, entryPoint: EntryPoint, formatPath: string, isCore: boolean, - format: EntryPointFormat, transformDts: boolean, pathMappings?: PathMappings, + fs: FileSystem, entryPoint: EntryPoint, transformCache: TransformCache, formatPath: string, + isCore: boolean, format: EntryPointFormat, transformDts: boolean, pathMappings?: PathMappings, mirrorDtsFromSrc: boolean = false, enableI18nLegacyMessageIdFormat: boolean = true): EntryPointBundle { // Create the TS program and necessary helpers. const rootDir = entryPoint.packagePath; const options: ts .CompilerOptions = {allowJs: true, maxNodeModuleJsDepth: Infinity, rootDir, ...pathMappings}; - const srcHost = new NgccSourcesCompilerHost(fs, options, entryPoint.packagePath); - const dtsHost = new NgtscCompilerHost(fs, options); + const entryPointCache = new EntryPointCache(fs, transformCache); + const dtsHost = new NgccDtsCompilerHost(fs, options, entryPointCache); + const srcHost = new NgccSourcesCompilerHost(fs, options, entryPointCache, entryPoint.packagePath); // Create the bundle programs, as necessary. const absFormatPath = fs.resolve(entryPoint.path, formatPath); diff --git a/packages/compiler-cli/ngcc/src/packages/ngcc_compiler_host.ts b/packages/compiler-cli/ngcc/src/packages/ngcc_compiler_host.ts index 4c5d62f767e3d8..3527bdc535eb6c 100644 --- a/packages/compiler-cli/ngcc/src/packages/ngcc_compiler_host.ts +++ b/packages/compiler-cli/ngcc/src/packages/ngcc_compiler_host.ts @@ -10,6 +10,7 @@ import * as ts from 'typescript'; import {AbsoluteFsPath, FileSystem, NgtscCompilerHost} from '../../../src/ngtsc/file_system'; import {isWithinPackage} from '../analysis/util'; import {isRelativePath} from '../utils'; +import {EntryPointCache} from './transform_cache'; /** * Represents a compiler host that resolves a module import as a JavaScript source file if @@ -18,19 +19,23 @@ import {isRelativePath} from '../utils'; * would otherwise let TypeScript prefer the .d.ts file instead of the JavaScript source file. */ export class NgccSourcesCompilerHost extends NgtscCompilerHost { - private cache = ts.createModuleResolutionCache( - this.getCurrentDirectory(), file => this.getCanonicalFileName(file)); - - constructor(fs: FileSystem, options: ts.CompilerOptions, protected packagePath: AbsoluteFsPath) { + constructor( + fs: FileSystem, options: ts.CompilerOptions, private cache: EntryPointCache, + protected packagePath: AbsoluteFsPath) { super(fs, options); } + getSourceFile(fileName: string, languageVersion: ts.ScriptTarget): ts.SourceFile|undefined { + return this.cache.getCachedSourceFile(fileName, languageVersion); + } + resolveModuleNames( moduleNames: string[], containingFile: string, reusedNames?: string[], redirectedReference?: ts.ResolvedProjectReference): Array { return moduleNames.map(moduleName => { const {resolvedModule} = ts.resolveModuleName( - moduleName, containingFile, this.options, this, this.cache, redirectedReference); + moduleName, containingFile, this.options, this, this.cache.moduleResolutionCache, + redirectedReference); // If the module request originated from a relative import in a JavaScript source file, // TypeScript may have resolved the module to its .d.ts declaration file if the .js source @@ -59,3 +64,29 @@ export class NgccSourcesCompilerHost extends NgtscCompilerHost { }); } } + +/** + * A compiler host implementation that is used for the typings program. It leverages the entry-point + * cache for source files and module resolution, as these results can be reused across the sources + * program. + */ +export class NgccDtsCompilerHost extends NgtscCompilerHost { + constructor(fs: FileSystem, options: ts.CompilerOptions, private cache: EntryPointCache) { + super(fs, options); + } + + getSourceFile(fileName: string, languageVersion: ts.ScriptTarget): ts.SourceFile|undefined { + return this.cache.getCachedSourceFile(fileName, languageVersion); + } + + resolveModuleNames( + moduleNames: string[], containingFile: string, reusedNames?: string[], + redirectedReference?: ts.ResolvedProjectReference): Array { + return moduleNames.map(moduleName => { + const {resolvedModule} = ts.resolveModuleName( + moduleName, containingFile, this.options, this, this.cache.moduleResolutionCache, + redirectedReference); + return resolvedModule; + }); + } +} diff --git a/packages/compiler-cli/ngcc/src/packages/transform_cache.ts b/packages/compiler-cli/ngcc/src/packages/transform_cache.ts new file mode 100644 index 00000000000000..6273a72000d093 --- /dev/null +++ b/packages/compiler-cli/ngcc/src/packages/transform_cache.ts @@ -0,0 +1,96 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import * as ts from 'typescript'; +import {AbsoluteFsPath, basename, FileSystem} from '../../../src/ngtsc/file_system'; + +/** + * A cache that holds on to data that can be shared for processing all entry-points in a single + * invocation of ngcc. In particular, the default library files are cached as parsed `ts.SourceFile` + * as these files are used in each entry-point and some are expensive to parse, especially + * `lib.es5.d.ts` and `lib.dom.d.ts`. Additionally, a `ts.ModuleResolutionCache` is exposed for + * all module resolution operations to use, such that all entry-points can leverage a single module + * resolution cache. + * + * The lifecycle of this cache corresponds with a single invocation of ngcc. Separate invocations, + * e.g. the CLI's synchronous module resolution fallback will therefore all have their own cache. + * This is because module resolution results cannot be assumed to be valid across invocations, as + * modifications of the file-system may have invalidated earlier results. + */ +export class TransformCache { + private defaultLibCache = new Map(); + readonly moduleResolutionCache: ts.ModuleResolutionCache; + + constructor(private fs: FileSystem) { + this.moduleResolutionCache = ts.createModuleResolutionCache(fs.pwd(), fileName => { + return fs.isCaseSensitive() ? fileName : fileName.toLowerCase(); + }); + } + + getCachedSourceFile(fileName: string): ts.SourceFile|undefined { + if (/^lib\..+\.d\.ts$/.test(basename(fileName))) { + return this.getDefaultLibFileCached(fileName); + } else { + return undefined; + } + } + + private getDefaultLibFileCached(fileName: string): ts.SourceFile|undefined { + const absPath = this.fs.resolve(fileName); + if (!this.defaultLibCache.has(absPath)) { + const content = readFile(absPath, this.fs); + if (content === undefined) { + return undefined; + } + const sf = ts.createSourceFile(fileName, content, ts.ScriptTarget.ES2015); + this.defaultLibCache.set(absPath, sf); + } + return this.defaultLibCache.get(absPath)!; + } +} + +/** + * A cache for processing a single entry-point. This exists to share `ts.SourceFile`s between the + * source and typing programs that are created for a single program. Additionally, it leverages the + * transform cache for module resolution and sharing of default library files. + */ +export class EntryPointCache { + private readonly sfCache = new Map(); + + constructor(private fs: FileSystem, private transformCache: TransformCache) {} + + get moduleResolutionCache(): ts.ModuleResolutionCache { + return this.transformCache.moduleResolutionCache; + } + + getCachedSourceFile(fileName: string, languageVersion: ts.ScriptTarget): ts.SourceFile|undefined { + const staticSf = this.transformCache.getCachedSourceFile(fileName); + if (staticSf !== undefined) { + return staticSf; + } + + const absPath = this.fs.resolve(fileName); + if (this.sfCache.has(absPath)) { + return this.sfCache.get(absPath); + } + + const content = readFile(absPath, this.fs); + if (content === undefined) { + return undefined; + } + const sf = ts.createSourceFile(fileName, content, languageVersion); + this.sfCache.set(absPath, sf); + return sf; + } +} + +function readFile(absPath: AbsoluteFsPath, fs: FileSystem): string|undefined { + if (!fs.exists(absPath) || !fs.stat(absPath).isFile()) { + return undefined; + } + return fs.readFile(absPath); +} diff --git a/packages/compiler-cli/ngcc/test/helpers/utils.ts b/packages/compiler-cli/ngcc/test/helpers/utils.ts index 75a23805543c58..a7316031eb77e0 100644 --- a/packages/compiler-cli/ngcc/test/helpers/utils.ts +++ b/packages/compiler-cli/ngcc/test/helpers/utils.ts @@ -14,6 +14,7 @@ import {NgccEntryPointConfig} from '../../src/packages/configuration'; import {EntryPoint, EntryPointFormat} from '../../src/packages/entry_point'; import {EntryPointBundle} from '../../src/packages/entry_point_bundle'; import {NgccSourcesCompilerHost} from '../../src/packages/ngcc_compiler_host'; +import {EntryPointCache, TransformCache} from '../../src/packages/transform_cache'; export type TestConfig = Pick; @@ -68,7 +69,8 @@ export function makeTestBundleProgram( const rootDir = fs.dirname(entryPointPath); const options: ts.CompilerOptions = {allowJs: true, maxNodeModuleJsDepth: Infinity, checkJs: false, rootDir, rootDirs: [rootDir]}; - const host = new NgccSourcesCompilerHost(fs, options, rootDir); + const entryPointCache = new EntryPointCache(fs, new TransformCache(fs)); + const host = new NgccSourcesCompilerHost(fs, options, entryPointCache, rootDir); return makeBundleProgram( fs, isCore, rootDir, path, 'r3_symbols.js', options, host, additionalFiles); } diff --git a/packages/compiler-cli/ngcc/test/packages/entry_point_bundle_spec.ts b/packages/compiler-cli/ngcc/test/packages/entry_point_bundle_spec.ts index 8eb133ed0a23a9..b6ed88befa9b5e 100644 --- a/packages/compiler-cli/ngcc/test/packages/entry_point_bundle_spec.ts +++ b/packages/compiler-cli/ngcc/test/packages/entry_point_bundle_spec.ts @@ -10,6 +10,7 @@ import {runInEachFileSystem} from '../../../src/ngtsc/file_system/testing'; import {loadTestFiles} from '../../../test/helpers'; import {EntryPoint} from '../../src/packages/entry_point'; import {makeEntryPointBundle} from '../../src/packages/entry_point_bundle'; +import {TransformCache} from '../../src/packages/transform_cache'; runInEachFileSystem(() => { describe('entry point bundle', () => { @@ -180,7 +181,8 @@ runInEachFileSystem(() => { ignoreMissingDependencies: false, generateDeepReexports: false, }; - const esm5bundle = makeEntryPointBundle(fs, entryPoint, './index.js', false, 'esm5', true); + const esm5bundle = makeEntryPointBundle( + fs, entryPoint, new TransformCache(fs), './index.js', false, 'esm5', true); expect(esm5bundle.src.program.getSourceFiles().map(sf => sf.fileName)) .toEqual(jasmine.arrayWithExactContents([ @@ -292,7 +294,8 @@ runInEachFileSystem(() => { generateDeepReexports: false, }; const esm5bundle = makeEntryPointBundle( - fs, entryPoint, './index.js', false, 'esm5', /* transformDts */ true, + fs, entryPoint, new TransformCache(fs), './index.js', false, 'esm5', + /* transformDts */ true, /* pathMappings */ undefined, /* mirrorDtsFromSrc */ true); expect(esm5bundle.src.program.getSourceFiles().map(sf => _(sf.fileName))) @@ -329,7 +332,8 @@ runInEachFileSystem(() => { generateDeepReexports: false, }; const esm5bundle = makeEntryPointBundle( - fs, entryPoint, './index.js', false, 'esm5', /* transformDts */ true, + fs, entryPoint, new TransformCache(fs), './index.js', false, 'esm5', + /* transformDts */ true, /* pathMappings */ undefined, /* mirrorDtsFromSrc */ true); expect(esm5bundle.src.program.getSourceFiles().map(sf => sf.fileName)) .toContain(absoluteFrom('/node_modules/test/internal.js')); @@ -352,7 +356,8 @@ runInEachFileSystem(() => { generateDeepReexports: false, }; const esm5bundle = makeEntryPointBundle( - fs, entryPoint, './esm2015/index.js', false, 'esm2015', /* transformDts */ true, + fs, entryPoint, new TransformCache(fs), './esm2015/index.js', false, 'esm2015', + /* transformDts */ true, /* pathMappings */ undefined, /* mirrorDtsFromSrc */ true); expect(esm5bundle.src.program.getSourceFiles().map(sf => sf.fileName)) .toContain(absoluteFrom('/node_modules/internal/esm2015/src/internal.js')); @@ -375,7 +380,8 @@ runInEachFileSystem(() => { generateDeepReexports: false, }; const esm5bundle = makeEntryPointBundle( - fs, entryPoint, './index.js', false, 'esm5', /* transformDts */ true, + fs, entryPoint, new TransformCache(fs), './index.js', false, 'esm5', + /* transformDts */ true, /* pathMappings */ undefined, /* mirrorDtsFromSrc */ false); expect(esm5bundle.src.program.getSourceFiles().map(sf => sf.fileName)) .toContain(absoluteFrom('/node_modules/test/internal.js')); @@ -399,7 +405,8 @@ runInEachFileSystem(() => { generateDeepReexports: false, }; const bundle = makeEntryPointBundle( - fs, entryPoint, './index.js', false, 'esm2015', /* transformDts */ true, + fs, entryPoint, new TransformCache(fs), './index.js', false, 'esm2015', + /* transformDts */ true, /* pathMappings */ undefined, /* mirrorDtsFromSrc */ true); expect(bundle.rootDirs).toEqual([absoluteFrom('/node_modules/primary')]); }); diff --git a/packages/compiler-cli/ngcc/test/writing/new_entry_point_file_writer_spec.ts b/packages/compiler-cli/ngcc/test/writing/new_entry_point_file_writer_spec.ts index b8fb4aba3382ba..9f48b1fdc17642 100644 --- a/packages/compiler-cli/ngcc/test/writing/new_entry_point_file_writer_spec.ts +++ b/packages/compiler-cli/ngcc/test/writing/new_entry_point_file_writer_spec.ts @@ -12,6 +12,7 @@ import {loadTestFiles} from '../../../test/helpers'; import {NgccConfiguration} from '../../src/packages/configuration'; import {EntryPoint, EntryPointFormat, EntryPointJsonProperty, getEntryPointInfo, isEntryPoint} from '../../src/packages/entry_point'; import {EntryPointBundle, makeEntryPointBundle} from '../../src/packages/entry_point_bundle'; +import {TransformCache} from '../../src/packages/transform_cache'; import {FileWriter} from '../../src/writing/file_writer'; import {NewEntryPointFileWriter} from '../../src/writing/new_entry_point_file_writer'; import {DirectPackageJsonUpdater} from '../../src/writing/package_json_updater'; @@ -635,6 +636,7 @@ runInEachFileSystem(() => { fs: FileSystem, entryPoint: EntryPoint, formatProperty: EntryPointJsonProperty, format: EntryPointFormat): EntryPointBundle { return makeEntryPointBundle( - fs, entryPoint, entryPoint.packageJson[formatProperty]!, false, format, true); + fs, entryPoint, new TransformCache(fs), entryPoint.packageJson[formatProperty]!, false, + format, true); } });