diff --git a/packages/devkit/src/utils/convert-nx-executor.ts b/packages/devkit/src/utils/convert-nx-executor.ts index 3fcfd6bdc8b48..1e6f8c956e805 100644 --- a/packages/devkit/src/utils/convert-nx-executor.ts +++ b/packages/devkit/src/utils/convert-nx-executor.ts @@ -8,6 +8,7 @@ const { Workspaces, readNxJsonFromDisk, retrieveProjectConfigurationsWithAngularProjects, + shutdownPluginWorkers, } = requireNx(); /** @@ -38,6 +39,7 @@ export function convertNxExecutor(executor: Executor) { (workspaces as any).readProjectsConfigurations({ _includeProjectsFromAngularJson: true, }); + shutdownPluginWorkers?.(); const context: ExecutorContext = { root: builderContext.workspaceRoot, diff --git a/packages/nx/src/config/workspaces.spec.ts b/packages/nx/src/config/workspaces.spec.ts index 7f1c2f2e42792..9e229bb6fb5e0 100644 --- a/packages/nx/src/config/workspaces.spec.ts +++ b/packages/nx/src/config/workspaces.spec.ts @@ -3,7 +3,7 @@ import { TempFs } from '../internal-testing-utils/temp-fs'; import { withEnvironmentVariables } from '../internal-testing-utils/with-environment'; import { retrieveProjectConfigurations } from '../project-graph/utils/retrieve-workspace-files'; import { readNxJson } from './configuration'; -import { loadNxPluginsInIsolation } from '../project-graph/plugins/internal-api'; +import { shutdownPluginWorkers } from '../project-graph/plugins/plugin-pool'; describe('Workspaces', () => { let fs: TempFs; @@ -37,19 +37,7 @@ describe('Workspaces', () => { { NX_WORKSPACE_ROOT_PATH: fs.tempDir, }, - async () => { - const [plugins, cleanup] = await loadNxPluginsInIsolation( - readNxJson(fs.tempDir).plugins, - fs.tempDir - ); - const res = retrieveProjectConfigurations( - plugins, - fs.tempDir, - readNxJson(fs.tempDir) - ); - cleanup(); - return res; - } + () => retrieveProjectConfigurations(fs.tempDir, readNxJson(fs.tempDir)) ); expect(projects['my-package']).toEqual({ name: 'my-package', diff --git a/packages/nx/src/daemon/server/handle-request-project-graph.ts b/packages/nx/src/daemon/server/handle-request-project-graph.ts index 1dc4507383c82..449d57a693805 100644 --- a/packages/nx/src/daemon/server/handle-request-project-graph.ts +++ b/packages/nx/src/daemon/server/handle-request-project-graph.ts @@ -3,8 +3,6 @@ import { serializeResult } from '../socket-utils'; import { serverLogger } from './logger'; import { getCachedSerializedProjectGraphPromise } from './project-graph-incremental-recomputation'; import { HandlerResult } from './server'; -import { getPlugins } from './plugins'; -import { readNxJson } from '../../config/nx-json'; export async function handleRequestProjectGraph(): Promise { try { diff --git a/packages/nx/src/daemon/server/plugins.ts b/packages/nx/src/daemon/server/plugins.ts deleted file mode 100644 index ddf57fdcb393c..0000000000000 --- a/packages/nx/src/daemon/server/plugins.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { readNxJson } from '../../config/nx-json'; -import { - RemotePlugin, - loadNxPluginsInIsolation, -} from '../../project-graph/plugins/internal-api'; -import { workspaceRoot } from '../../utils/workspace-root'; - -let loadedPlugins: Promise; -let cleanup: () => void; - -export async function getPlugins() { - if (loadedPlugins) { - return loadedPlugins; - } - const pluginsConfiguration = readNxJson().plugins ?? []; - const [result, cleanupFn] = await loadNxPluginsInIsolation( - pluginsConfiguration, - workspaceRoot - ); - cleanup = cleanupFn; - return result; -} - -export function cleanupPlugins() { - cleanup(); -} diff --git a/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts b/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts index 94a55b5ae2f0e..05c59fa50b04b 100644 --- a/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts +++ b/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts @@ -29,8 +29,6 @@ import { workspaceRoot } from '../../utils/workspace-root'; import { notifyFileWatcherSockets } from './file-watching/file-watcher-sockets'; import { serverLogger } from './logger'; import { NxWorkspaceFilesExternals } from '../../native'; -import { RemotePlugin } from '../../project-graph/plugins/internal-api'; -import { getPlugins } from './plugins'; interface SerializedProjectGraph { error: Error | null; @@ -71,15 +69,14 @@ export async function getCachedSerializedProjectGraphPromise(): Promise 0) { @@ -202,9 +199,7 @@ async function processCollectedUpdatedAndDeletedFiles( } } -async function processFilesAndCreateAndSerializeProjectGraph( - plugins: RemotePlugin[] -): Promise { +async function processFilesAndCreateAndSerializeProjectGraph(): Promise { try { performance.mark('hash-watched-changes-start'); const updatedFiles = [...collectedUpdatedFiles.values()]; @@ -224,7 +219,6 @@ async function processFilesAndCreateAndSerializeProjectGraph( const nxJson = readNxJson(workspaceRoot); global.NX_GRAPH_CREATION = true; const graphNodes = await retrieveProjectConfigurations( - plugins, workspaceRoot, nxJson ); @@ -281,8 +275,7 @@ async function createAndSerializeProjectGraph({ allWorkspaceFiles, rustReferences, currentProjectFileMapCache || readFileMapCache(), - true, - await getPlugins() + true ); currentProjectFileMapCache = projectFileMapCache; currentProjectGraph = projectGraph; diff --git a/packages/nx/src/daemon/server/shutdown-utils.ts b/packages/nx/src/daemon/server/shutdown-utils.ts index 2a53e907f699d..22affc1b46a84 100644 --- a/packages/nx/src/daemon/server/shutdown-utils.ts +++ b/packages/nx/src/daemon/server/shutdown-utils.ts @@ -4,26 +4,21 @@ import { serverLogger } from './logger'; import { serializeResult } from '../socket-utils'; import { deleteDaemonJsonProcessCache } from '../cache'; import type { Watcher } from '../../native'; -import { cleanupPlugins } from './plugins'; export const SERVER_INACTIVITY_TIMEOUT_MS = 10800000 as const; // 10800000 ms = 3 hours let watcherInstance: Watcher | undefined; - export function storeWatcherInstance(instance: Watcher) { watcherInstance = instance; } - export function getWatcherInstance() { return watcherInstance; } let outputWatcherInstance: Watcher | undefined; - export function storeOutputWatcherInstance(instance: Watcher) { outputWatcherInstance = instance; } - export function getOutputWatcherInstance() { return outputWatcherInstance; } @@ -40,7 +35,6 @@ export async function handleServerProcessTermination({ try { server.close(); deleteDaemonJsonProcessCache(); - cleanupPlugins(); if (watcherInstance) { await watcherInstance.stop(); diff --git a/packages/nx/src/devkit-internals.ts b/packages/nx/src/devkit-internals.ts index a24b33c98670a..cdb257674d0b4 100644 --- a/packages/nx/src/devkit-internals.ts +++ b/packages/nx/src/devkit-internals.ts @@ -21,3 +21,4 @@ export { findProjectForPath, } from './project-graph/utils/find-project-for-path'; export { registerTsProject } from './plugins/js/utils/register'; +export { shutdownPluginWorkers } from './project-graph/plugins/plugin-pool'; diff --git a/packages/nx/src/executors/utils/convert-nx-executor.ts b/packages/nx/src/executors/utils/convert-nx-executor.ts index 76d0d9dbd3322..40b2566d91c10 100644 --- a/packages/nx/src/executors/utils/convert-nx-executor.ts +++ b/packages/nx/src/executors/utils/convert-nx-executor.ts @@ -7,7 +7,6 @@ import { readNxJson } from '../../config/nx-json'; import { Executor, ExecutorContext } from '../../config/misc-interfaces'; import { retrieveProjectConfigurations } from '../../project-graph/utils/retrieve-workspace-files'; import { ProjectsConfigurations } from '../../config/workspace-json-project-json'; -import { loadNxPluginsInIsolation } from '../../project-graph/plugins/internal-api'; /** * Convert an Nx Executor into an Angular Devkit Builder @@ -18,22 +17,15 @@ export function convertNxExecutor(executor: Executor) { const builderFunction = (options, builderContext) => { const promise = async () => { const nxJsonConfiguration = readNxJson(builderContext.workspaceRoot); - - const [plugins, cleanup] = await loadNxPluginsInIsolation( - nxJsonConfiguration.plugins, - builderContext.workspaceRoot - ); const projectsConfigurations: ProjectsConfigurations = { version: 2, projects: ( await retrieveProjectConfigurations( - plugins, builderContext.workspaceRoot, nxJsonConfiguration ) ).projects, }; - cleanup(); const context: ExecutorContext = { root: builderContext.workspaceRoot, projectName: builderContext.target.project, diff --git a/packages/nx/src/generators/utils/project-configuration.ts b/packages/nx/src/generators/utils/project-configuration.ts index 1f93b6a07aa75..8a702193b22d5 100644 --- a/packages/nx/src/generators/utils/project-configuration.ts +++ b/packages/nx/src/generators/utils/project-configuration.ts @@ -28,7 +28,6 @@ import { readJson, writeJson } from './json'; import { readNxJson } from './nx-json'; import type { Tree } from '../tree'; -import { NxPlugin } from '../../project-graph/plugins'; export { readNxJson, updateNxJson } from './nx-json'; @@ -202,7 +201,7 @@ function readAndCombineAllProjectConfigurations(tree: Tree): { ]; const projectGlobPatterns = configurationGlobs([ ProjectJsonProjectsPlugin, - { createNodes: packageJsonWorkspacesCreateNodes } as NxPlugin, + { createNodes: packageJsonWorkspacesCreateNodes }, ]); const globbedFiles = globWithWorkspaceContext(tree.root, projectGlobPatterns); const createdFiles = findCreatedProjectFiles(tree, patterns); diff --git a/packages/nx/src/migrations/update-15-1-0/set-project-names.ts b/packages/nx/src/migrations/update-15-1-0/set-project-names.ts index 01ca3b2afde80..f1b6d7d855453 100644 --- a/packages/nx/src/migrations/update-15-1-0/set-project-names.ts +++ b/packages/nx/src/migrations/update-15-1-0/set-project-names.ts @@ -4,14 +4,16 @@ import { dirname } from 'path'; import { readJson, writeJson } from '../../generators/utils/json'; import { formatChangedFilesWithPrettierIfAvailable } from '../../generators/internal-utils/format-changed-files-with-prettier-if-available'; import { retrieveProjectConfigurationPaths } from '../../project-graph/utils/retrieve-workspace-files'; -import { loadPlugins } from '../../project-graph/plugins/internal-api'; +import { loadNxPlugins } from '../../project-graph/plugins/internal-api'; +import { shutdownPluginWorkers } from '../../project-graph/plugins/plugin-pool'; export default async function (tree: Tree) { const nxJson = readNxJson(tree); - const projectFiles = retrieveProjectConfigurationPaths( + const projectFiles = await retrieveProjectConfigurationPaths( tree.root, - (await loadPlugins(nxJson?.plugins ?? [], tree.root)).map((p) => p.plugin) + await loadNxPlugins(nxJson?.plugins) ); + await shutdownPluginWorkers(); const projectJsons = projectFiles.filter((f) => f.endsWith('project.json')); for (let f of projectJsons) { diff --git a/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.spec.ts b/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.spec.ts index 2c79667bdac9e..a4b8d1b52dc69 100644 --- a/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.spec.ts +++ b/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.spec.ts @@ -1,5 +1,4 @@ import { TempFs } from '../../../../internal-testing-utils/temp-fs'; - const tempFs = new TempFs('explicit-project-deps'); import { ProjectGraphBuilder } from '../../../../project-graph/project-graph-builder'; @@ -10,8 +9,7 @@ import { } from '../../../../project-graph/utils/retrieve-workspace-files'; import { CreateDependenciesContext } from '../../../../project-graph/plugins'; import { setupWorkspaceContext } from '../../../../utils/workspace-context'; -import ProjectJsonProjectsPlugin from '../../../project-json/build-nodes/project-json'; -import { loadNxPluginsInIsolation } from '../../../../project-graph/plugins/internal-api'; +import { shutdownPluginWorkers } from '../../../../project-graph/plugins/plugin-pool'; // projectName => tsconfig import path const dependencyProjectNamesToImportPaths = { @@ -25,6 +23,10 @@ describe('explicit project dependencies', () => { tempFs.reset(); }); + afterEach(async () => { + await shutdownPluginWorkers(); + }); + describe('static imports, dynamic imports, and commonjs requires', () => { it('should build explicit dependencies for static imports, and top-level dynamic imports and commonjs requires', async () => { const source = 'proj'; @@ -566,13 +568,10 @@ async function createContext( setupWorkspaceContext(tempFs.tempDir); - const [plugins, cleanup] = await loadNxPluginsInIsolation([], tempFs.tempDir); const { projects, projectRootMap } = await retrieveProjectConfigurations( - plugins, tempFs.tempDir, nxJson ); - cleanup(); const { fileMap } = await retrieveWorkspaceFiles( tempFs.tempDir, diff --git a/packages/nx/src/project-graph/affected/locators/project-glob-changes.ts b/packages/nx/src/project-graph/affected/locators/project-glob-changes.ts index aaabf121c1ff5..32c126cc8b813 100644 --- a/packages/nx/src/project-graph/affected/locators/project-glob-changes.ts +++ b/packages/nx/src/project-graph/affected/locators/project-glob-changes.ts @@ -4,14 +4,13 @@ import { workspaceRoot } from '../../../utils/workspace-root'; import { join } from 'path'; import { existsSync } from 'fs'; import { configurationGlobs } from '../../utils/retrieve-workspace-files'; -import { loadPlugins } from '../../plugins/internal-api'; +import { loadNxPlugins } from '../../plugins/internal-api'; import { combineGlobPatterns } from '../../../utils/globs'; export const getTouchedProjectsFromProjectGlobChanges: TouchedProjectLocator = async (touchedFiles, projectGraphNodes, nxJson): Promise => { - const plugins = await loadPlugins(nxJson?.plugins ?? [], workspaceRoot); const globPattern = combineGlobPatterns( - configurationGlobs(plugins.map((p) => p.plugin)) + configurationGlobs(await loadNxPlugins(nxJson?.plugins, workspaceRoot)) ); const touchedProjects = new Set(); diff --git a/packages/nx/src/project-graph/build-project-graph.ts b/packages/nx/src/project-graph/build-project-graph.ts index 11f7addb46fd7..f0a985ec8c313 100644 --- a/packages/nx/src/project-graph/build-project-graph.ts +++ b/packages/nx/src/project-graph/build-project-graph.ts @@ -13,7 +13,7 @@ import { } from './nx-deps-cache'; import { applyImplicitDependencies } from './utils/implicit-project-dependencies'; import { normalizeProjectNodes } from './utils/normalize-project-nodes'; -import { RemotePlugin } from './plugins/internal-api'; +import { loadNxPlugins } from './plugins/internal-api'; import { isNxPluginV1, isNxPluginV2 } from './plugins/utils'; import { CreateDependenciesContext } from './plugins'; import { getRootTsConfigPath } from '../plugins/js/utils/typescript'; @@ -29,8 +29,10 @@ import { ProjectConfiguration } from '../config/workspace-json-project-json'; import { readNxJson } from '../config/configuration'; import { existsSync } from 'fs'; import { PackageJson } from '../utils/package-json'; +import { getNxRequirePaths } from '../utils/installation-directory'; import { output } from '../utils/output'; -import { NxWorkspaceFilesExternals } from '../native'; +import { ExternalObject, NxWorkspaceFilesExternals } from '../native'; +import { shutdownPluginWorkers } from './plugins/plugin-pool'; let storedFileMap: FileMap | null = null; let storedAllWorkspaceFiles: FileData[] | null = null; @@ -66,8 +68,7 @@ export async function buildProjectGraphUsingProjectFileMap( allWorkspaceFiles: FileData[], rustReferences: NxWorkspaceFilesExternals, fileMapCache: FileMapCache | null, - shouldWriteCache: boolean, - plugins: RemotePlugin[] + shouldWriteCache: boolean ): Promise<{ projectGraph: ProjectGraph; projectFileMapCache: FileMapCache; @@ -117,8 +118,7 @@ export async function buildProjectGraphUsingProjectFileMap( externalNodes, context, cachedFileData, - projectGraphVersion, - plugins + projectGraphVersion ); const projectFileMapCache = createProjectFileMapCache( nxJson, @@ -129,6 +129,7 @@ export async function buildProjectGraphUsingProjectFileMap( if (shouldWriteCache) { writeCache(projectFileMapCache, projectGraph); } + await shutdownPluginWorkers(); return { projectGraph, projectFileMapCache, @@ -164,8 +165,7 @@ async function buildProjectGraphUsingContext( knownExternalNodes: Record, ctx: CreateDependenciesContext, cachedFileData: CachedFileData, - projectGraphVersion: string, - plugins: RemotePlugin[] + projectGraphVersion: string ) { performance.mark('build project graph:start'); @@ -178,7 +178,7 @@ async function buildProjectGraphUsingContext( await normalizeProjectNodes(ctx, builder); const initProjectGraph = builder.getUpdatedProjectGraph(); - const r = await updateProjectGraphWithPlugins(ctx, initProjectGraph, plugins); + const r = await updateProjectGraphWithPlugins(ctx, initProjectGraph); const updatedBuilder = new ProjectGraphBuilder(r, ctx.fileMap.projectFileMap); for (const proj of Object.keys(cachedFileData.projectFileMap)) { @@ -235,9 +235,12 @@ function createContext( async function updateProjectGraphWithPlugins( context: CreateDependenciesContext, - initProjectGraph: ProjectGraph, - plugins: RemotePlugin[] + initProjectGraph: ProjectGraph ) { + const plugins = await loadNxPlugins( + context.nxJsonConfiguration?.plugins, + context.workspaceRoot + ); let graph = initProjectGraph; for (const plugin of plugins) { try { diff --git a/packages/nx/src/project-graph/plugins/internal-api.ts b/packages/nx/src/project-graph/plugins/internal-api.ts index a6fe1ceaeb7ba..8cb558638b91f 100644 --- a/packages/nx/src/project-graph/plugins/internal-api.ts +++ b/packages/nx/src/project-graph/plugins/internal-api.ts @@ -12,11 +12,10 @@ import { loadRemoteNxPlugin } from './plugin-pool'; import { CreateNodesContext, CreateNodesResult, + NxPlugin, NxPluginV2, } from './public-api'; -export { loadPlugins, loadPlugin } from './worker-api'; - export type CreateNodesResultWithContext = CreateNodesResult & { file: string; pluginName: string; @@ -45,16 +44,12 @@ export type RemotePlugin = // holding resolved nx plugin objects. // Allows loaded plugins to not be reloaded when // referenced multiple times. -export const nxPluginCache: Map, () => void]> = - new Map(); +export const nxPluginCache: Map> = new Map(); -/** - * This loads plugins in isolation in their own worker so that they do not disturb other workers or the main process. - */ -export async function loadNxPluginsInIsolation( +export async function loadNxPlugins( plugins: PluginConfiguration[], root = workspaceRoot -): Promise<[RemotePlugin[], () => void]> { +): Promise { const result: Promise[] = []; plugins ??= []; @@ -69,39 +64,26 @@ export async function loadNxPluginsInIsolation( // We push the nx core node plugins onto the end, s.t. it overwrites any other plugins plugins.push(...(await getDefaultPlugins(root))); - const cleanupFunctions: Array<() => void> = []; for (const plugin of plugins) { - const [loadedPluginPromise, cleanup] = loadNxPluginInIsolation( - plugin, - root - ); - result.push(loadedPluginPromise); - cleanupFunctions.push(cleanup); + result.push(loadNxPlugin(plugin, root)); } - return [ - await Promise.all(result), - () => { - for (const fn of cleanupFunctions) { - fn(); - } - }, - ]; + return Promise.all(result); } -export function loadNxPluginInIsolation( +export async function loadNxPlugin( plugin: PluginConfiguration, root = workspaceRoot -): [Promise, () => void] { +): Promise { const cacheKey = JSON.stringify(plugin); if (nxPluginCache.has(cacheKey)) { - return nxPluginCache.get(cacheKey); + return await nxPluginCache.get(cacheKey)!; } - const [loadingPlugin, cleanup] = loadRemoteNxPlugin(plugin, root); - nxPluginCache.set(cacheKey, [loadingPlugin, cleanup]); - return [loadingPlugin, cleanup]; + const loadingPlugin = loadRemoteNxPlugin(plugin, root); + nxPluginCache.set(cacheKey, loadingPlugin); + return await loadingPlugin; } export async function getDefaultPlugins(root: string) { diff --git a/packages/nx/src/project-graph/plugins/plugin-pool.ts b/packages/nx/src/project-graph/plugins/plugin-pool.ts index 376671e48b3d5..1c20608c5560a 100644 --- a/packages/nx/src/project-graph/plugins/plugin-pool.ts +++ b/packages/nx/src/project-graph/plugins/plugin-pool.ts @@ -9,16 +9,20 @@ import { PluginConfiguration } from '../../config/nx-json'; import { RemotePlugin, nxPluginCache } from './internal-api'; import { PluginWorkerResult, consumeMessage, createMessage } from './messaging'; -const cleanupFunctions = new Set<() => void>(); +const pool: Set = new Set(); -const pluginNames = new Map(); +const pidMap = new Map }>(); -interface PendingPromise { +interface PromiseBankEntry { promise: Promise; resolver: (result: any) => void; rejector: (err: any) => void; } +// transaction id (tx) -> Promise, Resolver, Rejecter +// Makes sure that we can resolve the correct promise when the worker sends back the result +const promiseBank = new Map(); + export function loadRemoteNxPlugin(plugin: PluginConfiguration, root: string) { // this should only really be true when running unit tests within // the Nx repo. We still need to start the worker in this case, @@ -43,64 +47,51 @@ export function loadRemoteNxPlugin(plugin: PluginConfiguration, root: string) { ], }); worker.send(createMessage({ type: 'load', payload: { plugin, root } })); + pool.add(worker); // logger.verbose(`[plugin-worker] started worker: ${worker.pid}`); - const pendingPromises = new Map(); - - const exitHandler = createWorkerExitHandler(worker, pendingPromises); - - const cleanupFunction = () => { - worker.off('exit', exitHandler); - shutdownPluginWorker(worker, pendingPromises); - }; - - cleanupFunctions.add(cleanupFunction); - - return [ - new Promise((res, rej) => { - worker.on( - 'message', - createWorkerHandler(worker, pendingPromises, res, rej) - ); - worker.on('exit', exitHandler); - }), - () => { - cleanupFunction(); - cleanupFunctions.delete(cleanupFunction); - }, - ] as const; + return new Promise((res, rej) => { + worker.on('message', createWorkerHandler(worker, res, rej)); + worker.on('exit', createWorkerExitHandler(worker)); + }); } -async function shutdownPluginWorker( - worker: ChildProcess, - pendingPromises: Map -) { +let pluginWorkersShutdown = false; + +export async function shutdownPluginWorkers() { // Clears the plugin cache so no refs to the workers are held nxPluginCache.clear(); + // Marks the workers as shutdown so that we don't report unexpected exits + pluginWorkersShutdown = true; + // logger.verbose(`[plugin-pool] starting worker shutdown`); - // Other things may be interacting with the worker. - // Wait for all pending promises to be done before killing the worker - await Promise.all( - Array.from(pendingPromises.values()).map(({ promise }) => promise) - ); + const pending = getPendingPromises(pool, pidMap); + + for (const pendingPromise of pending) { + pendingPromise.rejector(new Error('Shutting down')); + } + + // logger.verbose(`[plugin-pool] all pending operations completed`); + + for (const childProcess of pool) { + childProcess.kill('SIGINT'); + } - worker.kill('SIGINT'); + // logger.verbose(`[plugin-pool] all workers killed`); } /** * Creates a message handler for the given worker. * @param worker Instance of plugin-worker - * @param pending Set of pending promises * @param onload Resolver for RemotePlugin promise * @param onloadError Rejecter for RemotePlugin promise * @returns Function to handle messages from the worker */ function createWorkerHandler( worker: ChildProcess, - pending: Map, onload: (plugin: RemotePlugin) => void, onloadError: (err?: unknown) => void ) { @@ -118,7 +109,8 @@ function createWorkerHandler( if (result.success) { const { name, createNodesPattern } = result; pluginName = name; - pluginNames.set(worker, pluginName); + const pending = new Set(); + pidMap.set(worker.pid, { name, pending }); onload({ name, createNodes: createNodesPattern @@ -171,59 +163,80 @@ function createWorkerHandler( } }, createDependenciesResult: ({ tx, ...result }) => { - const { resolver, rejector } = pending.get(tx); + const { resolver, rejector } = promiseBank.get(tx); if (result.success) { resolver(result.dependencies); } else if (result.success === false) { rejector(result.error); } + pidMap.get(worker.pid)?.pending.delete(tx); + promiseBank.delete(tx); }, createNodesResult: ({ tx, ...result }) => { - const { resolver, rejector } = pending.get(tx); + const { resolver, rejector } = promiseBank.get(tx); if (result.success) { resolver(result.result); } else if (result.success === false) { rejector(result.error); } + pidMap.get(worker.pid)?.pending.delete(tx); + promiseBank.delete(tx); }, processProjectGraphResult: ({ tx, ...result }) => { - const { resolver, rejector } = pending.get(tx); + const { resolver, rejector } = promiseBank.get(tx); if (result.success) { resolver(result.graph); } else if (result.success === false) { rejector(result.error); } + pidMap.get(worker.pid)?.pending.delete(tx); + promiseBank.delete(tx); }, }); }; } -function createWorkerExitHandler( - worker: ChildProcess, - pendingPromises: Map -) { +function createWorkerExitHandler(worker: ChildProcess) { return () => { - for (const [_, pendingPromise] of pendingPromises) { - pendingPromise.rejector( - new Error( - `Plugin worker ${ - pluginNames.get(worker) ?? worker.pid - } exited unexpectedly with code ${worker.exitCode}` - ) - ); + if (!pluginWorkersShutdown) { + pidMap.get(worker.pid)?.pending.forEach((tx) => { + const { rejector } = promiseBank.get(tx); + rejector( + new Error( + `Plugin worker ${ + pidMap.get(worker.pid).name ?? worker.pid + } exited unexpectedly with code ${worker.exitCode}` + ) + ); + }); + shutdownPluginWorkers(); } }; } process.on('exit', () => { - for (const fn of cleanupFunctions) { - fn(); + if (pool.size) { + shutdownPluginWorkers(); } }); +function getPendingPromises( + pool: Set, + pidMap: Map }> +) { + const pendingTxs: Array = []; + for (const p of pool) { + const { pending } = pidMap.get(p.pid) ?? { pending: new Set() }; + for (const tx of pending) { + pendingTxs.push(promiseBank.get(tx)); + } + } + return pendingTxs; +} + function registerPendingPromise( tx: string, - pending: Map, + pending: Set, callback: () => void ): Promise { let resolver, rejector; @@ -233,15 +246,18 @@ function registerPendingPromise( rejector = rej; callback(); - }).finally(() => { + }).then((val) => { + // Remove the promise from the pending set pending.delete(tx); + // Return the original value + return val; }); - pending.set(tx, { + pending.add(tx); + promiseBank.set(tx, { promise, resolver, rejector, }); - return promise; } diff --git a/packages/nx/src/project-graph/plugins/plugin-worker.ts b/packages/nx/src/project-graph/plugins/plugin-worker.ts index 268a35a071eee..63c37e57ce282 100644 --- a/packages/nx/src/project-graph/plugins/plugin-worker.ts +++ b/packages/nx/src/project-graph/plugins/plugin-worker.ts @@ -1,8 +1,15 @@ -import { consumeMessage, PluginWorkerMessage } from './messaging'; -import { CreateNodesResultWithContext, NormalizedPlugin } from './internal-api'; +import { getNxRequirePaths } from '../../utils/installation-directory'; +import { loadNxPluginAsync } from './worker-api'; +import { PluginWorkerMessage, consumeMessage } from './messaging'; +import { PluginConfiguration } from '../../config/nx-json'; +import { ProjectConfiguration } from '../../config/workspace-json-project-json'; +import { retrieveProjectConfigurationsWithoutPluginInference } from '../utils/retrieve-workspace-files'; +import type { + CreateNodesResultWithContext, + NormalizedPlugin, +} from './internal-api'; import { CreateNodesContext } from './public-api'; import { CreateNodesError } from './utils'; -import { loadPlugin } from './worker-api'; global.NX_GRAPH_CREATION = true; @@ -14,7 +21,7 @@ process.on('message', async (message: string) => { load: async ({ plugin: pluginConfiguration, root }) => { process.chdir(root); try { - ({ plugin, options: pluginOptions } = await loadPlugin( + ({ plugin, options: pluginOptions } = await loadPluginFromWorker( pluginConfiguration, root )); @@ -87,6 +94,24 @@ process.on('message', async (message: string) => { }); }); +let projectsWithoutInference: Record; + +async function loadPluginFromWorker(plugin: PluginConfiguration, root: string) { + try { + require.resolve(typeof plugin === 'string' ? plugin : plugin.plugin); + } catch { + // If a plugin cannot be resolved, we will need projects to resolve it + projectsWithoutInference ??= + await retrieveProjectConfigurationsWithoutPluginInference(root); + } + return await loadNxPluginAsync( + plugin, + getNxRequirePaths(root), + projectsWithoutInference, + root + ); +} + function runCreateNodesInParallel( configFiles: string[], context: CreateNodesContext diff --git a/packages/nx/src/project-graph/plugins/worker-api.ts b/packages/nx/src/project-graph/plugins/worker-api.ts index 8b2750a0d9aa0..679c027b68302 100644 --- a/packages/nx/src/project-graph/plugins/worker-api.ts +++ b/packages/nx/src/project-graph/plugins/worker-api.ts @@ -1,6 +1,7 @@ // This file contains methods and utilities that should **only** be used by the plugin worker. import { ProjectConfiguration } from '../../config/workspace-json-project-json'; +import { PluginConfiguration } from '../../config/nx-json'; import { join } from 'node:path/posix'; import { getNxRequirePaths } from '../../utils/installation-directory'; @@ -9,6 +10,7 @@ import { readModulePackageJsonWithoutFallbacks, } from '../../utils/package-json'; import { readJsonFile } from '../../utils/fileutils'; +import path = require('node:path/posix'); import { workspaceRoot } from '../../utils/workspace-root'; import { existsSync } from 'node:fs'; import { readTsConfig } from '../../utils/typescript'; @@ -25,11 +27,42 @@ import { logger } from '../../utils/logger'; import type * as ts from 'typescript'; import { extname } from 'node:path'; -import { NxPlugin } from './public-api'; -import path = require('node:path/posix'); -import { PluginConfiguration } from '../../config/nx-json'; -import { retrieveProjectConfigurationsWithoutPluginInference } from '../utils/retrieve-workspace-files'; import { normalizeNxPlugin } from './utils'; +import { NxPlugin } from './public-api'; + +export type LoadedNxPlugin = { + plugin: NxPlugin; + options?: unknown; +}; + +export async function loadNxPluginAsync( + pluginConfiguration: PluginConfiguration, + paths: string[], + projects: Record, + root: string +): Promise { + const { plugin: moduleName, options } = + typeof pluginConfiguration === 'object' + ? pluginConfiguration + : { plugin: pluginConfiguration, options: undefined }; + + performance.mark(`Load Nx Plugin: ${moduleName} - start`); + let { pluginPath, name } = await getPluginPathAndName( + moduleName, + paths, + projects, + root + ); + const plugin = normalizeNxPlugin(await importPluginModule(pluginPath)); + plugin.name ??= name; + performance.mark(`Load Nx Plugin: ${moduleName} - end`); + performance.measure( + `Load Nx Plugin: ${moduleName}`, + `Load Nx Plugin: ${moduleName} - start`, + `Load Nx Plugin: ${moduleName} - end` + ); + return { plugin, options }; +} export function readPluginPackageJson( pluginName: string, @@ -219,65 +252,6 @@ export function getPluginPathAndName( return { pluginPath, name }; } -let projectsWithoutInference: Record; - -export async function loadPlugins( - plugins: PluginConfiguration[], - root: string -): Promise { - return await Promise.all(plugins.map((p) => loadPlugin(p, root))); -} - -export async function loadPlugin(plugin: PluginConfiguration, root: string) { - try { - require.resolve(typeof plugin === 'string' ? plugin : plugin.plugin); - } catch { - // If a plugin cannot be resolved, we will need projects to resolve it - projectsWithoutInference ??= - await retrieveProjectConfigurationsWithoutPluginInference(root); - } - return await loadNxPluginAsync( - plugin, - getNxRequirePaths(root), - projectsWithoutInference, - root - ); -} - -export type LoadedNxPlugin = { - plugin: NxPlugin; - options?: unknown; -}; - -export async function loadNxPluginAsync( - pluginConfiguration: PluginConfiguration, - paths: string[], - projects: Record, - root: string -): Promise { - const { plugin: moduleName, options } = - typeof pluginConfiguration === 'object' - ? pluginConfiguration - : { plugin: pluginConfiguration, options: undefined }; - - performance.mark(`Load Nx Plugin: ${moduleName} - start`); - let { pluginPath, name } = await getPluginPathAndName( - moduleName, - paths, - projects, - root - ); - const plugin = normalizeNxPlugin(await importPluginModule(pluginPath)); - plugin.name ??= name; - performance.mark(`Load Nx Plugin: ${moduleName} - end`); - performance.measure( - `Load Nx Plugin: ${moduleName}`, - `Load Nx Plugin: ${moduleName} - start`, - `Load Nx Plugin: ${moduleName} - end` - ); - return { plugin, options }; -} - async function importPluginModule(pluginPath: string): Promise { const m = await import(pluginPath); if ( diff --git a/packages/nx/src/project-graph/project-graph.ts b/packages/nx/src/project-graph/project-graph.ts index 812c58cdb4553..ff2c0eac8ccad 100644 --- a/packages/nx/src/project-graph/project-graph.ts +++ b/packages/nx/src/project-graph/project-graph.ts @@ -17,7 +17,6 @@ import { retrieveWorkspaceFiles, } from './utils/retrieve-workspace-files'; import { readNxJson } from '../config/nx-json'; -import { loadNxPluginsInIsolation, RemotePlugin } from './plugins/internal-api'; /** * Synchronously reads the latest cached copy of the workspace's ProjectGraph. @@ -81,11 +80,9 @@ export async function buildProjectGraphAndSourceMapsWithoutDaemon() { global.NX_GRAPH_CREATION = true; const nxJson = readNxJson(); - const [plugins, cleanup] = await loadNxPluginsInIsolation(nxJson.plugins); - performance.mark('retrieve-project-configurations:start'); const { projects, externalNodes, sourceMaps, projectRootMap } = - await retrieveProjectConfigurations(plugins, workspaceRoot, nxJson); + await retrieveProjectConfigurations(workspaceRoot, nxJson); performance.mark('retrieve-project-configurations:end'); performance.mark('retrieve-workspace-files:start'); @@ -103,13 +100,11 @@ export async function buildProjectGraphAndSourceMapsWithoutDaemon() { allWorkspaceFiles, rustReferences, cacheEnabled ? readFileMapCache() : null, - cacheEnabled, - plugins + cacheEnabled ) ).projectGraph; performance.mark('build-project-graph-using-project-file-map:end'); delete global.NX_GRAPH_CREATION; - cleanup(); return { projectGraph, sourceMaps }; } diff --git a/packages/nx/src/project-graph/utils/project-configuration-utils.ts b/packages/nx/src/project-graph/utils/project-configuration-utils.ts index 05f08e16c542c..c609b7d9b7c0c 100644 --- a/packages/nx/src/project-graph/utils/project-configuration-utils.ts +++ b/packages/nx/src/project-graph/utils/project-configuration-utils.ts @@ -19,6 +19,7 @@ import { CreateNodesResultWithContext, RemotePlugin, } from '../plugins/internal-api'; +import { shutdownPluginWorkers } from '../plugins/plugin-pool'; export type SourceInformation = [file: string, plugin: string]; export type ConfigurationSourceMaps = Record< @@ -229,7 +230,11 @@ export function buildProjectsConfigurationsFromProjectPathsAndPlugins( let r = createNodes(matchedFiles, { nxJsonConfiguration: nxJson, workspaceRoot: root, - }); + }).catch((e) => + shutdownPluginWorkers().then(() => { + throw e; + }) + ); results.push(r); } diff --git a/packages/nx/src/project-graph/utils/retrieve-workspace-files.spec.ts b/packages/nx/src/project-graph/utils/retrieve-workspace-files.spec.ts index fc49dacc6b305..8ed64564f6f8d 100644 --- a/packages/nx/src/project-graph/utils/retrieve-workspace-files.spec.ts +++ b/packages/nx/src/project-graph/utils/retrieve-workspace-files.spec.ts @@ -25,17 +25,9 @@ describe('retrieveProjectConfigurationPaths', () => { }) ); - const configPaths = retrieveProjectConfigurationPaths(fs.tempDir, [ + const configPaths = await retrieveProjectConfigurationPaths(fs.tempDir, [ { - name: 'test', - createNodes: [ - '{project.json,**/project.json}', - () => { - return { - projects: {}, - }; - }, - ], + createNodes: ['{project.json,**/project.json}', () => {}], }, ]); diff --git a/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts b/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts index 428533d895acd..34061f6173ad2 100644 --- a/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts +++ b/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts @@ -10,17 +10,14 @@ import { buildProjectsConfigurationsFromProjectPathsAndPlugins, ConfigurationSourceMaps, } from './project-configuration-utils'; -import { - RemotePlugin, - loadNxPluginsInIsolation, -} from '../plugins/internal-api'; +import { RemotePlugin, loadNxPlugins } from '../plugins/internal-api'; import { getNxWorkspaceFilesFromContext, globWithWorkspaceContext, } from '../../utils/workspace-context'; import { buildAllWorkspaceFiles } from './build-all-workspace-files'; import { join } from 'path'; -import { NxPlugin } from '../plugins'; +import { shutdownPluginWorkers } from '../plugins/plugin-pool'; /** * Walks the workspace directory to create the `projectFileMap`, `ProjectConfigurations` and `allWorkspaceFiles` @@ -63,17 +60,23 @@ export async function retrieveWorkspaceFiles( /** * Walk through the workspace and return `ProjectConfigurations`. Only use this if the projectFileMap is not needed. + * + * @param workspaceRoot + * @param nxJson */ export async function retrieveProjectConfigurations( - plugins: RemotePlugin[], workspaceRoot: string, nxJson: NxJsonConfiguration ): Promise { + const plugins = await loadNxPlugins(nxJson?.plugins ?? [], workspaceRoot); const projects = await _retrieveProjectConfigurations( workspaceRoot, nxJson, plugins ); + if (!global.NX_GRAPH_CREATION) { + await shutdownPluginWorkers(); + } return projects; } @@ -94,18 +97,9 @@ export async function retrieveProjectConfigurationsWithAngularProjects( pluginsToLoad.push(join(__dirname, '../../adapter/angular-json')); } - const [plugins, cleanup] = await loadNxPluginsInIsolation( - nxJson?.plugins ?? [], - workspaceRoot - ); + const plugins = await loadNxPlugins(nxJson?.plugins ?? [], workspaceRoot); - const res = _retrieveProjectConfigurations( - workspaceRoot, - nxJson, - await plugins - ); - cleanup(); - return res; + return _retrieveProjectConfigurations(workspaceRoot, nxJson, plugins); } export type RetrievedGraphNodes = { @@ -133,7 +127,7 @@ function _retrieveProjectConfigurations( export function retrieveProjectConfigurationPaths( root: string, - plugins: NxPlugin[] + plugins: PluginGlobsOnly ): string[] { const projectGlobPatterns = configurationGlobs(plugins); return globWithWorkspaceContext(root, projectGlobPatterns); @@ -149,7 +143,7 @@ export async function retrieveProjectConfigurationsWithoutPluginInference( root: string ): Promise> { const nxJson = readNxJson(root); - const [plugins, cleanup] = await loadNxPluginsInIsolation([]); // only load default plugins + const plugins = await loadNxPlugins([]); // only load default plugins const projectGlobPatterns = retrieveProjectConfigurationPaths(root, plugins); const cacheKey = root + ',' + projectGlobPatterns.join(','); @@ -167,8 +161,6 @@ export async function retrieveProjectConfigurationsWithoutPluginInference( projectsWithoutPluginCache.set(cacheKey, projects); - cleanup(); - return projects; } @@ -203,10 +195,12 @@ export async function createProjectConfigurations( }; } -export function configurationGlobs(plugins: Array): string[] { +type PluginGlobsOnly = Array<{ createNodes?: readonly [string, ...unknown[]] }>; + +export function configurationGlobs(plugins: PluginGlobsOnly): string[] { const globPatterns = []; for (const plugin of plugins) { - if ('createNodes' in plugin && plugin.createNodes) { + if (plugin.createNodes) { globPatterns.push(plugin.createNodes[0]); } } diff --git a/packages/nx/src/utils/plugins/plugin-capabilities.ts b/packages/nx/src/utils/plugins/plugin-capabilities.ts index 3d9fdb22eae03..6b105847768b8 100644 --- a/packages/nx/src/utils/plugins/plugin-capabilities.ts +++ b/packages/nx/src/utils/plugins/plugin-capabilities.ts @@ -2,8 +2,9 @@ import * as chalk from 'chalk'; import { dirname, join } from 'path'; import { ProjectConfiguration } from '../../config/workspace-json-project-json'; -import { NxPlugin, readPluginPackageJson } from '../../project-graph/plugins'; -import { loadPlugin } from '../../project-graph/plugins/internal-api'; +import { readPluginPackageJson } from '../../project-graph/plugins'; +import { RemotePlugin } from '../../project-graph/plugins/internal-api'; +import { loadRemoteNxPlugin } from '../../project-graph/plugins/plugin-pool'; import { readJsonFile } from '../fileutils'; import { getNxRequirePaths } from '../installation-directory'; import { output } from '../output'; @@ -45,7 +46,7 @@ export async function getPluginCapabilities( getNxRequirePaths(workspaceRoot) ); const pluginModule = includeRuntimeCapabilities - ? await tryGetModule(packageJson, workspaceRoot) + ? await tryGetModule(packageJson, workspaceRoot, projects) : ({} as Record); return { name: pluginName, @@ -98,18 +99,19 @@ export async function getPluginCapabilities( async function tryGetModule( packageJson: PackageJson, - workspaceRoot: string -): Promise { + workspaceRoot: string, + projects: Record +): Promise { try { return packageJson.generators ?? packageJson.executors ?? packageJson['nx-migrations'] ?? packageJson['schematics'] ?? packageJson['builders'] - ? (await loadPlugin(packageJson.name, workspaceRoot)).plugin + ? await loadRemoteNxPlugin(packageJson.name, workspaceRoot) : ({ name: packageJson.name, - } as NxPlugin); + } as RemotePlugin); } catch { return null; }