diff --git a/packages/compiler-cli/ngcc/src/main.ts b/packages/compiler-cli/ngcc/src/main.ts index d87bd556cd096..b6d74e19df4d2 100644 --- a/packages/compiler-cli/ngcc/src/main.ts +++ b/packages/compiler-cli/ngcc/src/main.ts @@ -8,8 +8,6 @@ /// -import * as os from 'os'; - import {AbsoluteFsPath, FileSystem, resolve} from '../../src/ngtsc/file_system'; import {Logger} from '../../src/ngtsc/logging'; import {ParsedConfiguration} from '../../src/perform_compile'; @@ -35,7 +33,7 @@ import {composeTaskCompletedCallbacks, createLogErrorHandler, createMarkAsProces import {AsyncLocker} from './locking/async_locker'; import {LockFileWithChildProcess} from './locking/lock_file_with_child_process'; import {SyncLocker} from './locking/sync_locker'; -import {AsyncNgccOptions, getSharedSetup, SyncNgccOptions} from './ngcc_options'; +import {AsyncNgccOptions, getMaxNumberOfWorkers, getSharedSetup, SyncNgccOptions} from './ngcc_options'; import {NgccConfiguration} from './packages/configuration'; import {EntryPointJsonProperty, SUPPORTED_FORMAT_PROPERTIES} from './packages/entry_point'; import {EntryPointManifest, InvalidatingEntryPointManifest} from './packages/entry_point_manifest'; @@ -92,10 +90,9 @@ export function mainNgcc(options: AsyncNgccOptions|SyncNgccOptions): void|Promis return; } - // Execute in parallel, if async execution is acceptable and there are more than 2 CPU cores. - // (One CPU core is always reserved for the master process and we need at least 2 worker processes - // in order to run tasks in parallel.) - const inParallel = async && (os.cpus().length > 2); + // Determine the number of workers to use and whether ngcc should run in parallel. + const workerCount = async ? getMaxNumberOfWorkers() : 1; + const inParallel = workerCount > 1; const analyzeEntryPoints = getAnalyzeEntryPointsFn( logger, finder, fileSystem, supportedPropertiesToConsider, compileAllFormats, @@ -113,7 +110,7 @@ export function mainNgcc(options: AsyncNgccOptions|SyncNgccOptions): void|Promis const createTaskCompletedCallback = getCreateTaskCompletedCallback(pkgJsonUpdater, errorOnFailedEntryPoint, logger, fileSystem); const executor = getExecutor( - async, inParallel, logger, fileWriter, pkgJsonUpdater, fileSystem, config, + async, workerCount, logger, fileWriter, pkgJsonUpdater, fileSystem, config, createTaskCompletedCallback); return executor.execute(analyzeEntryPoints, createCompileFn); @@ -153,7 +150,7 @@ function getCreateTaskCompletedCallback( } function getExecutor( - async: boolean, inParallel: boolean, logger: Logger, fileWriter: FileWriter, + async: boolean, workerCount: number, logger: Logger, fileWriter: FileWriter, pkgJsonUpdater: PackageJsonUpdater, fileSystem: FileSystem, config: NgccConfiguration, createTaskCompletedCallback: CreateTaskCompletedCallback): Executor { const lockFile = new LockFileWithChildProcess(fileSystem, logger); @@ -161,9 +158,8 @@ function getExecutor( // Execute asynchronously (either serially or in parallel) const {retryAttempts, retryDelay} = config.getLockingConfig(); const locker = new AsyncLocker(lockFile, logger, retryDelay, retryAttempts); - if (inParallel) { - // Execute in parallel. Use up to 8 CPU cores for workers, always reserving one for master. - const workerCount = Math.min(8, os.cpus().length - 1); + if (workerCount > 1) { + // Execute in parallel. return new ClusterExecutor( workerCount, fileSystem, logger, fileWriter, pkgJsonUpdater, locker, createTaskCompletedCallback); diff --git a/packages/compiler-cli/ngcc/src/ngcc_options.ts b/packages/compiler-cli/ngcc/src/ngcc_options.ts index 7ef38ec650f80..52fb3ab4d8cfc 100644 --- a/packages/compiler-cli/ngcc/src/ngcc_options.ts +++ b/packages/compiler-cli/ngcc/src/ngcc_options.ts @@ -5,6 +5,8 @@ * Use of this source code is governed by an MIT-style license that can be * found in the LICENSE file at https://angular.io/license */ +import * as os from 'os'; + import {absoluteFrom, AbsoluteFsPath, FileSystem, getFileSystem} from '../../src/ngtsc/file_system'; import {ConsoleLogger, Logger, LogLevel} from '../../src/ngtsc/logging'; import {ParsedConfiguration, readConfiguration} from '../../src/perform_compile'; @@ -254,3 +256,26 @@ function checkForSolutionStyleTsConfig( ` ngcc ... --tsconfig "${fileSystem.relative(projectPath, tsConfig.project)}"`); } } + +/** + * Determines the maximum number of workers to use for parallel execution. This can be set using the + * NGCC_MAX_WORKERS environment variable, or is computed based on the number of available CPUs. One + * CPU core is always reserved for the master process, so we take the number of CPUs minus one, with + * a maximum of 4 workers. We don't scale the number of workers beyond 4 by default, as it takes + * considerably more memory and CPU cycles while not offering a substantial improvement in time. + */ +export function getMaxNumberOfWorkers(): number { + const maxWorkers = process.env.NGCC_MAX_WORKERS; + if (maxWorkers === undefined) { + // Use up to 4 CPU cores for workers, always reserving one for master. + return Math.max(1, Math.min(4, os.cpus().length - 1)); + } + + const numericMaxWorkers = +maxWorkers.trim(); + if (!Number.isInteger(numericMaxWorkers)) { + throw new Error('NGCC_MAX_WORKERS should be an integer.'); + } else if (numericMaxWorkers < 1) { + throw new Error('NGCC_MAX_WORKERS should be at least 1.'); + } + return numericMaxWorkers; +} diff --git a/packages/compiler-cli/ngcc/test/ngcc_options_spec.ts b/packages/compiler-cli/ngcc/test/ngcc_options_spec.ts index 56fa85b7eb3a7..1ecb30b037955 100644 --- a/packages/compiler-cli/ngcc/test/ngcc_options_spec.ts +++ b/packages/compiler-cli/ngcc/test/ngcc_options_spec.ts @@ -5,12 +5,13 @@ * Use of this source code is governed by an MIT-style license that can be * found in the LICENSE file at https://angular.io/license */ +import * as os from 'os'; import {absoluteFrom, AbsoluteFsPath, FileSystem, getFileSystem} from '../../src/ngtsc/file_system'; import {runInEachFileSystem} from '../../src/ngtsc/file_system/testing'; import {MockLogger} from '../../src/ngtsc/logging/testing'; -import {clearTsConfigCache, getSharedSetup, NgccOptions} from '../src/ngcc_options'; +import {clearTsConfigCache, getMaxNumberOfWorkers, getSharedSetup, NgccOptions} from '../src/ngcc_options'; @@ -100,6 +101,67 @@ runInEachFileSystem(() => { }); }); + describe('getMaxNumberOfWorkers', () => { + let processEnv: NodeJS.ProcessEnv; + let cpuSpy: jasmine.Spy; + beforeEach(() => { + processEnv = process.env; + process.env = {...process.env}; + cpuSpy = spyOn(os, 'cpus'); + }); + afterEach(() => { + process.env = processEnv; + }); + + it('should use NGCC_MAX_WORKERS environment variable if set', () => { + process.env.NGCC_MAX_WORKERS = '16'; + expect(getMaxNumberOfWorkers()).toBe(16); + process.env.NGCC_MAX_WORKERS = '8'; + expect(getMaxNumberOfWorkers()).toBe(8); + process.env.NGCC_MAX_WORKERS = ' 8 '; + expect(getMaxNumberOfWorkers()).toBe(8); + }); + + it('should throw an error if NGCC_MAX_WORKERS is less than 1', () => { + process.env.NGCC_MAX_WORKERS = '0'; + expect(() => getMaxNumberOfWorkers()) + .toThrow(new Error('NGCC_MAX_WORKERS should be at least 1.')); + process.env.NGCC_MAX_WORKERS = '-1'; + expect(() => getMaxNumberOfWorkers()) + .toThrow(new Error('NGCC_MAX_WORKERS should be at least 1.')); + }); + + it('should throw an error if NGCC_MAX_WORKERS is not an integer', () => { + process.env.NGCC_MAX_WORKERS = 'a'; + expect(() => getMaxNumberOfWorkers()) + .toThrow(new Error('NGCC_MAX_WORKERS should be an integer.')); + process.env.NGCC_MAX_WORKERS = '1.5'; + expect(() => getMaxNumberOfWorkers()) + .toThrow(new Error('NGCC_MAX_WORKERS should be an integer.')); + process.env.NGCC_MAX_WORKERS = '-'; + expect(() => getMaxNumberOfWorkers()) + .toThrow(new Error('NGCC_MAX_WORKERS should be an integer.')); + }); + + it('should fallback to the number of cpus, minus one (for the master process), with a maximum of 4 workers', + () => { + simulateNumberOfCpus(1); + expect(getMaxNumberOfWorkers()).toBe(1); + simulateNumberOfCpus(2); + expect(getMaxNumberOfWorkers()).toBe(1); + simulateNumberOfCpus(4); + expect(getMaxNumberOfWorkers()).toBe(3); + simulateNumberOfCpus(6); + expect(getMaxNumberOfWorkers()).toBe(4); + simulateNumberOfCpus(8); + expect(getMaxNumberOfWorkers()).toBe(4); + }); + + function simulateNumberOfCpus(cpus: number): void { + cpuSpy.and.returnValue(new Array(cpus).fill({model: 'Mock CPU'} as any)); + } + }); + /** * This function creates an object that contains the minimal required properties for NgccOptions. */