From ea364660600b46dca8641d93189d993bc520c49b Mon Sep 17 00:00:00 2001 From: JoostK Date: Mon, 14 Sep 2020 14:19:30 +0200 Subject: [PATCH] perf(ngcc): reduce maximum worker count (#38840) Recent optimizations to ngcc have significantly reduced the total time it takes to process `node_modules`, to such extend that sharding across multiple processes has become less effective. Previously, running ngcc asynchronously would allow for up to 8 workers to be allocated, however these workers have to repeat work that could otherwise be shared. Because ngcc is now able to reuse more shared computations, the overhead of multiple workers is increased and therefore becomes less effective. As an additional benefit, having fewer workers requires less memory and less startup time. To give an idea, using the following test setup: ```bash npx @angular/cli new perf-test cd perf-test yarn ng add @angular/material ./node_modules/.bin/ngcc --properties es2015 module main \ --first-only --create-ivy-entry-points ``` We observe the following figures on CI: | | 10.1.1 | PR #38840 | | ----------------- | --------- | --------- | | Sync | 85s | 25s | | Async (8 workers) | 22s | 16s | | Async (4 workers) | - | 11s | In addition to changing the default number of workers, ngcc will now use the environment variable `NGCC_MAX_WORKERS` that may be configured to either reduce or increase the number of workers. PR Close #38840 --- packages/compiler-cli/ngcc/src/main.ts | 20 +++--- .../compiler-cli/ngcc/src/ngcc_options.ts | 25 ++++++++ .../ngcc/test/ngcc_options_spec.ts | 64 ++++++++++++++++++- 3 files changed, 96 insertions(+), 13 deletions(-) diff --git a/packages/compiler-cli/ngcc/src/main.ts b/packages/compiler-cli/ngcc/src/main.ts index d87bd556cd096..b6d74e19df4d2 100644 --- a/packages/compiler-cli/ngcc/src/main.ts +++ b/packages/compiler-cli/ngcc/src/main.ts @@ -8,8 +8,6 @@ /// -import * as os from 'os'; - import {AbsoluteFsPath, FileSystem, resolve} from '../../src/ngtsc/file_system'; import {Logger} from '../../src/ngtsc/logging'; import {ParsedConfiguration} from '../../src/perform_compile'; @@ -35,7 +33,7 @@ import {composeTaskCompletedCallbacks, createLogErrorHandler, createMarkAsProces import {AsyncLocker} from './locking/async_locker'; import {LockFileWithChildProcess} from './locking/lock_file_with_child_process'; import {SyncLocker} from './locking/sync_locker'; -import {AsyncNgccOptions, getSharedSetup, SyncNgccOptions} from './ngcc_options'; +import {AsyncNgccOptions, getMaxNumberOfWorkers, getSharedSetup, SyncNgccOptions} from './ngcc_options'; import {NgccConfiguration} from './packages/configuration'; import {EntryPointJsonProperty, SUPPORTED_FORMAT_PROPERTIES} from './packages/entry_point'; import {EntryPointManifest, InvalidatingEntryPointManifest} from './packages/entry_point_manifest'; @@ -92,10 +90,9 @@ export function mainNgcc(options: AsyncNgccOptions|SyncNgccOptions): void|Promis return; } - // Execute in parallel, if async execution is acceptable and there are more than 2 CPU cores. - // (One CPU core is always reserved for the master process and we need at least 2 worker processes - // in order to run tasks in parallel.) - const inParallel = async && (os.cpus().length > 2); + // Determine the number of workers to use and whether ngcc should run in parallel. + const workerCount = async ? getMaxNumberOfWorkers() : 1; + const inParallel = workerCount > 1; const analyzeEntryPoints = getAnalyzeEntryPointsFn( logger, finder, fileSystem, supportedPropertiesToConsider, compileAllFormats, @@ -113,7 +110,7 @@ export function mainNgcc(options: AsyncNgccOptions|SyncNgccOptions): void|Promis const createTaskCompletedCallback = getCreateTaskCompletedCallback(pkgJsonUpdater, errorOnFailedEntryPoint, logger, fileSystem); const executor = getExecutor( - async, inParallel, logger, fileWriter, pkgJsonUpdater, fileSystem, config, + async, workerCount, logger, fileWriter, pkgJsonUpdater, fileSystem, config, createTaskCompletedCallback); return executor.execute(analyzeEntryPoints, createCompileFn); @@ -153,7 +150,7 @@ function getCreateTaskCompletedCallback( } function getExecutor( - async: boolean, inParallel: boolean, logger: Logger, fileWriter: FileWriter, + async: boolean, workerCount: number, logger: Logger, fileWriter: FileWriter, pkgJsonUpdater: PackageJsonUpdater, fileSystem: FileSystem, config: NgccConfiguration, createTaskCompletedCallback: CreateTaskCompletedCallback): Executor { const lockFile = new LockFileWithChildProcess(fileSystem, logger); @@ -161,9 +158,8 @@ function getExecutor( // Execute asynchronously (either serially or in parallel) const {retryAttempts, retryDelay} = config.getLockingConfig(); const locker = new AsyncLocker(lockFile, logger, retryDelay, retryAttempts); - if (inParallel) { - // Execute in parallel. Use up to 8 CPU cores for workers, always reserving one for master. - const workerCount = Math.min(8, os.cpus().length - 1); + if (workerCount > 1) { + // Execute in parallel. return new ClusterExecutor( workerCount, fileSystem, logger, fileWriter, pkgJsonUpdater, locker, createTaskCompletedCallback); diff --git a/packages/compiler-cli/ngcc/src/ngcc_options.ts b/packages/compiler-cli/ngcc/src/ngcc_options.ts index 7ef38ec650f80..52fb3ab4d8cfc 100644 --- a/packages/compiler-cli/ngcc/src/ngcc_options.ts +++ b/packages/compiler-cli/ngcc/src/ngcc_options.ts @@ -5,6 +5,8 @@ * Use of this source code is governed by an MIT-style license that can be * found in the LICENSE file at https://angular.io/license */ +import * as os from 'os'; + import {absoluteFrom, AbsoluteFsPath, FileSystem, getFileSystem} from '../../src/ngtsc/file_system'; import {ConsoleLogger, Logger, LogLevel} from '../../src/ngtsc/logging'; import {ParsedConfiguration, readConfiguration} from '../../src/perform_compile'; @@ -254,3 +256,26 @@ function checkForSolutionStyleTsConfig( ` ngcc ... --tsconfig "${fileSystem.relative(projectPath, tsConfig.project)}"`); } } + +/** + * Determines the maximum number of workers to use for parallel execution. This can be set using the + * NGCC_MAX_WORKERS environment variable, or is computed based on the number of available CPUs. One + * CPU core is always reserved for the master process, so we take the number of CPUs minus one, with + * a maximum of 4 workers. We don't scale the number of workers beyond 4 by default, as it takes + * considerably more memory and CPU cycles while not offering a substantial improvement in time. + */ +export function getMaxNumberOfWorkers(): number { + const maxWorkers = process.env.NGCC_MAX_WORKERS; + if (maxWorkers === undefined) { + // Use up to 4 CPU cores for workers, always reserving one for master. + return Math.max(1, Math.min(4, os.cpus().length - 1)); + } + + const numericMaxWorkers = +maxWorkers.trim(); + if (!Number.isInteger(numericMaxWorkers)) { + throw new Error('NGCC_MAX_WORKERS should be an integer.'); + } else if (numericMaxWorkers < 1) { + throw new Error('NGCC_MAX_WORKERS should be at least 1.'); + } + return numericMaxWorkers; +} diff --git a/packages/compiler-cli/ngcc/test/ngcc_options_spec.ts b/packages/compiler-cli/ngcc/test/ngcc_options_spec.ts index 56fa85b7eb3a7..1ecb30b037955 100644 --- a/packages/compiler-cli/ngcc/test/ngcc_options_spec.ts +++ b/packages/compiler-cli/ngcc/test/ngcc_options_spec.ts @@ -5,12 +5,13 @@ * Use of this source code is governed by an MIT-style license that can be * found in the LICENSE file at https://angular.io/license */ +import * as os from 'os'; import {absoluteFrom, AbsoluteFsPath, FileSystem, getFileSystem} from '../../src/ngtsc/file_system'; import {runInEachFileSystem} from '../../src/ngtsc/file_system/testing'; import {MockLogger} from '../../src/ngtsc/logging/testing'; -import {clearTsConfigCache, getSharedSetup, NgccOptions} from '../src/ngcc_options'; +import {clearTsConfigCache, getMaxNumberOfWorkers, getSharedSetup, NgccOptions} from '../src/ngcc_options'; @@ -100,6 +101,67 @@ runInEachFileSystem(() => { }); }); + describe('getMaxNumberOfWorkers', () => { + let processEnv: NodeJS.ProcessEnv; + let cpuSpy: jasmine.Spy; + beforeEach(() => { + processEnv = process.env; + process.env = {...process.env}; + cpuSpy = spyOn(os, 'cpus'); + }); + afterEach(() => { + process.env = processEnv; + }); + + it('should use NGCC_MAX_WORKERS environment variable if set', () => { + process.env.NGCC_MAX_WORKERS = '16'; + expect(getMaxNumberOfWorkers()).toBe(16); + process.env.NGCC_MAX_WORKERS = '8'; + expect(getMaxNumberOfWorkers()).toBe(8); + process.env.NGCC_MAX_WORKERS = ' 8 '; + expect(getMaxNumberOfWorkers()).toBe(8); + }); + + it('should throw an error if NGCC_MAX_WORKERS is less than 1', () => { + process.env.NGCC_MAX_WORKERS = '0'; + expect(() => getMaxNumberOfWorkers()) + .toThrow(new Error('NGCC_MAX_WORKERS should be at least 1.')); + process.env.NGCC_MAX_WORKERS = '-1'; + expect(() => getMaxNumberOfWorkers()) + .toThrow(new Error('NGCC_MAX_WORKERS should be at least 1.')); + }); + + it('should throw an error if NGCC_MAX_WORKERS is not an integer', () => { + process.env.NGCC_MAX_WORKERS = 'a'; + expect(() => getMaxNumberOfWorkers()) + .toThrow(new Error('NGCC_MAX_WORKERS should be an integer.')); + process.env.NGCC_MAX_WORKERS = '1.5'; + expect(() => getMaxNumberOfWorkers()) + .toThrow(new Error('NGCC_MAX_WORKERS should be an integer.')); + process.env.NGCC_MAX_WORKERS = '-'; + expect(() => getMaxNumberOfWorkers()) + .toThrow(new Error('NGCC_MAX_WORKERS should be an integer.')); + }); + + it('should fallback to the number of cpus, minus one (for the master process), with a maximum of 4 workers', + () => { + simulateNumberOfCpus(1); + expect(getMaxNumberOfWorkers()).toBe(1); + simulateNumberOfCpus(2); + expect(getMaxNumberOfWorkers()).toBe(1); + simulateNumberOfCpus(4); + expect(getMaxNumberOfWorkers()).toBe(3); + simulateNumberOfCpus(6); + expect(getMaxNumberOfWorkers()).toBe(4); + simulateNumberOfCpus(8); + expect(getMaxNumberOfWorkers()).toBe(4); + }); + + function simulateNumberOfCpus(cpus: number): void { + cpuSpy.and.returnValue(new Array(cpus).fill({model: 'Mock CPU'} as any)); + } + }); + /** * This function creates an object that contains the minimal required properties for NgccOptions. */