Revert "fix(core): only start plugin workers once (nrwl#22222)"

This reverts commit ec12e67.

packages/devkit/src/utils/convert-nx-executor.ts

@@ -8,6 +8,7 @@ const {
   Workspaces,
   readNxJsonFromDisk,
   retrieveProjectConfigurationsWithAngularProjects,
+  shutdownPluginWorkers,
 } = requireNx();
 
 /**
@@ -38,6 +39,7 @@ export function convertNxExecutor(executor: Executor) {
             (workspaces as any).readProjectsConfigurations({
               _includeProjectsFromAngularJson: true,
             });
+      shutdownPluginWorkers?.();
 
       const context: ExecutorContext = {
         root: builderContext.workspaceRoot,

packages/nx/src/config/workspaces.spec.ts

@@ -3,7 +3,7 @@ import { TempFs } from '../internal-testing-utils/temp-fs';
 import { withEnvironmentVariables } from '../internal-testing-utils/with-environment';
 import { retrieveProjectConfigurations } from '../project-graph/utils/retrieve-workspace-files';
 import { readNxJson } from './configuration';
-import { loadNxPluginsInIsolation } from '../project-graph/plugins/internal-api';
+import { shutdownPluginWorkers } from '../project-graph/plugins/plugin-pool';
 
 describe('Workspaces', () => {
   let fs: TempFs;
@@ -37,19 +37,7 @@ describe('Workspaces', () => {
         {
           NX_WORKSPACE_ROOT_PATH: fs.tempDir,
         },
-        async () => {
-          const [plugins, cleanup] = await loadNxPluginsInIsolation(
-            readNxJson(fs.tempDir).plugins,
-            fs.tempDir
-          );
-          const res = retrieveProjectConfigurations(
-            plugins,
-            fs.tempDir,
-            readNxJson(fs.tempDir)
-          );
-          cleanup();
-          return res;
-        }
+        () => retrieveProjectConfigurations(fs.tempDir, readNxJson(fs.tempDir))
       );
       expect(projects['my-package']).toEqual({
         name: 'my-package',

packages/nx/src/daemon/server/handle-request-project-graph.ts

@@ -3,8 +3,6 @@ import { serializeResult } from '../socket-utils';
 import { serverLogger } from './logger';
 import { getCachedSerializedProjectGraphPromise } from './project-graph-incremental-recomputation';
 import { HandlerResult } from './server';
-import { getPlugins } from './plugins';
-import { readNxJson } from '../../config/nx-json';
 
 export async function handleRequestProjectGraph(): Promise<HandlerResult> {
   try {

packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts

@@ -29,8 +29,6 @@ import { workspaceRoot } from '../../utils/workspace-root';
 import { notifyFileWatcherSockets } from './file-watching/file-watcher-sockets';
 import { serverLogger } from './logger';
 import { NxWorkspaceFilesExternals } from '../../native';
-import { RemotePlugin } from '../../project-graph/plugins/internal-api';
-import { getPlugins } from './plugins';
 
 interface SerializedProjectGraph {
   error: Error | null;
@@ -71,15 +69,14 @@ export async function getCachedSerializedProjectGraphPromise(): Promise<Serializ
     // reset the wait time
     waitPeriod = 100;
     await resetInternalStateIfNxDepsMissing();
-    const plugins = await getPlugins();
     if (collectedUpdatedFiles.size == 0 && collectedDeletedFiles.size == 0) {
       if (!cachedSerializedProjectGraphPromise) {
         cachedSerializedProjectGraphPromise =
-          processFilesAndCreateAndSerializeProjectGraph(plugins);
+          processFilesAndCreateAndSerializeProjectGraph();
       }
     } else {
       cachedSerializedProjectGraphPromise =
-        processFilesAndCreateAndSerializeProjectGraph(plugins);
+        processFilesAndCreateAndSerializeProjectGraph();
     }
     return await cachedSerializedProjectGraphPromise;
   } catch (e) {
@@ -126,7 +123,7 @@ export function addUpdatedAndDeletedFiles(
       }
 
       cachedSerializedProjectGraphPromise =
-        processFilesAndCreateAndSerializeProjectGraph(await getPlugins());
+        processFilesAndCreateAndSerializeProjectGraph();
       await cachedSerializedProjectGraphPromise;
 
       if (createdFiles.length > 0) {
@@ -202,9 +199,7 @@ async function processCollectedUpdatedAndDeletedFiles(
   }
 }
 
-async function processFilesAndCreateAndSerializeProjectGraph(
-  plugins: RemotePlugin[]
-): Promise<SerializedProjectGraph> {
+async function processFilesAndCreateAndSerializeProjectGraph(): Promise<SerializedProjectGraph> {
   try {
     performance.mark('hash-watched-changes-start');
     const updatedFiles = [...collectedUpdatedFiles.values()];
@@ -224,7 +219,6 @@ async function processFilesAndCreateAndSerializeProjectGraph(
     const nxJson = readNxJson(workspaceRoot);
     global.NX_GRAPH_CREATION = true;
     const graphNodes = await retrieveProjectConfigurations(
-      plugins,
       workspaceRoot,
       nxJson
     );
@@ -281,8 +275,7 @@ async function createAndSerializeProjectGraph({
         allWorkspaceFiles,
         rustReferences,
         currentProjectFileMapCache || readFileMapCache(),
-        true,
-        await getPlugins()
+        true
       );
     currentProjectFileMapCache = projectFileMapCache;
     currentProjectGraph = projectGraph;

packages/nx/src/daemon/server/shutdown-utils.ts

@@ -4,26 +4,21 @@ import { serverLogger } from './logger';
 import { serializeResult } from '../socket-utils';
 import { deleteDaemonJsonProcessCache } from '../cache';
 import type { Watcher } from '../../native';
-import { cleanupPlugins } from './plugins';
 
 export const SERVER_INACTIVITY_TIMEOUT_MS = 10800000 as const; // 10800000 ms = 3 hours
 
 let watcherInstance: Watcher | undefined;
-
 export function storeWatcherInstance(instance: Watcher) {
   watcherInstance = instance;
 }
-
 export function getWatcherInstance() {
   return watcherInstance;
 }
 
 let outputWatcherInstance: Watcher | undefined;
-
 export function storeOutputWatcherInstance(instance: Watcher) {
   outputWatcherInstance = instance;
 }
-
 export function getOutputWatcherInstance() {
   return outputWatcherInstance;
 }
@@ -40,7 +35,6 @@ export async function handleServerProcessTermination({
   try {
     server.close();
     deleteDaemonJsonProcessCache();
-    cleanupPlugins();
 
     if (watcherInstance) {
       await watcherInstance.stop();

packages/nx/src/devkit-internals.ts

@@ -21,3 +21,4 @@ export {
   findProjectForPath,
 } from './project-graph/utils/find-project-for-path';
 export { registerTsProject } from './plugins/js/utils/register';
+export { shutdownPluginWorkers } from './project-graph/plugins/plugin-pool';

packages/nx/src/executors/utils/convert-nx-executor.ts

@@ -7,7 +7,6 @@ import { readNxJson } from '../../config/nx-json';
 import { Executor, ExecutorContext } from '../../config/misc-interfaces';
 import { retrieveProjectConfigurations } from '../../project-graph/utils/retrieve-workspace-files';
 import { ProjectsConfigurations } from '../../config/workspace-json-project-json';
-import { loadNxPluginsInIsolation } from '../../project-graph/plugins/internal-api';
 
 /**
  * Convert an Nx Executor into an Angular Devkit Builder
@@ -18,22 +17,15 @@ export function convertNxExecutor(executor: Executor) {
   const builderFunction = (options, builderContext) => {
     const promise = async () => {
       const nxJsonConfiguration = readNxJson(builderContext.workspaceRoot);
-
-      const [plugins, cleanup] = await loadNxPluginsInIsolation(
-        nxJsonConfiguration.plugins,
-        builderContext.workspaceRoot
-      );
       const projectsConfigurations: ProjectsConfigurations = {
         version: 2,
         projects: (
           await retrieveProjectConfigurations(
-            plugins,
             builderContext.workspaceRoot,
             nxJsonConfiguration
           )
         ).projects,
       };
-      cleanup();
       const context: ExecutorContext = {
         root: builderContext.workspaceRoot,
         projectName: builderContext.target.project,

packages/nx/src/generators/utils/project-configuration.ts

@@ -28,7 +28,6 @@ import { readJson, writeJson } from './json';
 import { readNxJson } from './nx-json';
 
 import type { Tree } from '../tree';
-import { NxPlugin } from '../../project-graph/plugins';
 
 export { readNxJson, updateNxJson } from './nx-json';
 
@@ -202,7 +201,7 @@ function readAndCombineAllProjectConfigurations(tree: Tree): {
   ];
   const projectGlobPatterns = configurationGlobs([
     ProjectJsonProjectsPlugin,
-    { createNodes: packageJsonWorkspacesCreateNodes } as NxPlugin,
+    { createNodes: packageJsonWorkspacesCreateNodes },
   ]);
   const globbedFiles = globWithWorkspaceContext(tree.root, projectGlobPatterns);
   const createdFiles = findCreatedProjectFiles(tree, patterns);

packages/nx/src/migrations/update-15-1-0/set-project-names.ts

@@ -4,14 +4,16 @@ import { dirname } from 'path';
 import { readJson, writeJson } from '../../generators/utils/json';
 import { formatChangedFilesWithPrettierIfAvailable } from '../../generators/internal-utils/format-changed-files-with-prettier-if-available';
 import { retrieveProjectConfigurationPaths } from '../../project-graph/utils/retrieve-workspace-files';
-import { loadPlugins } from '../../project-graph/plugins/internal-api';
+import { loadNxPlugins } from '../../project-graph/plugins/internal-api';
+import { shutdownPluginWorkers } from '../../project-graph/plugins/plugin-pool';
 
 export default async function (tree: Tree) {
   const nxJson = readNxJson(tree);
-  const projectFiles = retrieveProjectConfigurationPaths(
+  const projectFiles = await retrieveProjectConfigurationPaths(
     tree.root,
-    (await loadPlugins(nxJson?.plugins ?? [], tree.root)).map((p) => p.plugin)
+    await loadNxPlugins(nxJson?.plugins)
   );
+  await shutdownPluginWorkers();
   const projectJsons = projectFiles.filter((f) => f.endsWith('project.json'));
 
   for (let f of projectJsons) {

packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.spec.ts

@@ -1,5 +1,4 @@
 import { TempFs } from '../../../../internal-testing-utils/temp-fs';
-
 const tempFs = new TempFs('explicit-project-deps');
 
 import { ProjectGraphBuilder } from '../../../../project-graph/project-graph-builder';
@@ -10,8 +9,7 @@ import {
 } from '../../../../project-graph/utils/retrieve-workspace-files';
 import { CreateDependenciesContext } from '../../../../project-graph/plugins';
 import { setupWorkspaceContext } from '../../../../utils/workspace-context';
-import ProjectJsonProjectsPlugin from '../../../project-json/build-nodes/project-json';
-import { loadNxPluginsInIsolation } from '../../../../project-graph/plugins/internal-api';
+import { shutdownPluginWorkers } from '../../../../project-graph/plugins/plugin-pool';
 
 // projectName => tsconfig import path
 const dependencyProjectNamesToImportPaths = {
@@ -25,6 +23,10 @@ describe('explicit project dependencies', () => {
     tempFs.reset();
   });
 
+  afterEach(async () => {
+    await shutdownPluginWorkers();
+  });
+
   describe('static imports, dynamic imports, and commonjs requires', () => {
     it('should build explicit dependencies for static imports, and top-level dynamic imports and commonjs requires', async () => {
       const source = 'proj';
@@ -566,13 +568,10 @@ async function createContext(
 
   setupWorkspaceContext(tempFs.tempDir);
 
-  const [plugins, cleanup] = await loadNxPluginsInIsolation([], tempFs.tempDir);
   const { projects, projectRootMap } = await retrieveProjectConfigurations(
-    plugins,
     tempFs.tempDir,
     nxJson
   );
-  cleanup();
 
   const { fileMap } = await retrieveWorkspaceFiles(
     tempFs.tempDir,

packages/nx/src/project-graph/affected/locators/project-glob-changes.ts

@@ -4,14 +4,13 @@ import { workspaceRoot } from '../../../utils/workspace-root';
 import { join } from 'path';
 import { existsSync } from 'fs';
 import { configurationGlobs } from '../../utils/retrieve-workspace-files';
-import { loadPlugins } from '../../plugins/internal-api';
+import { loadNxPlugins } from '../../plugins/internal-api';
 import { combineGlobPatterns } from '../../../utils/globs';
 
 export const getTouchedProjectsFromProjectGlobChanges: TouchedProjectLocator =
   async (touchedFiles, projectGraphNodes, nxJson): Promise<string[]> => {
-    const plugins = await loadPlugins(nxJson?.plugins ?? [], workspaceRoot);
     const globPattern = combineGlobPatterns(
-      configurationGlobs(plugins.map((p) => p.plugin))
+      configurationGlobs(await loadNxPlugins(nxJson?.plugins, workspaceRoot))
     );
 
     const touchedProjects = new Set<string>();

packages/nx/src/project-graph/build-project-graph.ts

@@ -13,7 +13,7 @@ import {
 } from './nx-deps-cache';
 import { applyImplicitDependencies } from './utils/implicit-project-dependencies';
 import { normalizeProjectNodes } from './utils/normalize-project-nodes';
-import { RemotePlugin } from './plugins/internal-api';
+import { loadNxPlugins } from './plugins/internal-api';
 import { isNxPluginV1, isNxPluginV2 } from './plugins/utils';
 import { CreateDependenciesContext } from './plugins';
 import { getRootTsConfigPath } from '../plugins/js/utils/typescript';
@@ -29,8 +29,10 @@ import { ProjectConfiguration } from '../config/workspace-json-project-json';
 import { readNxJson } from '../config/configuration';
 import { existsSync } from 'fs';
 import { PackageJson } from '../utils/package-json';
+import { getNxRequirePaths } from '../utils/installation-directory';
 import { output } from '../utils/output';
-import { NxWorkspaceFilesExternals } from '../native';
+import { ExternalObject, NxWorkspaceFilesExternals } from '../native';
+import { shutdownPluginWorkers } from './plugins/plugin-pool';
 
 let storedFileMap: FileMap | null = null;
 let storedAllWorkspaceFiles: FileData[] | null = null;
@@ -66,8 +68,7 @@ export async function buildProjectGraphUsingProjectFileMap(
   allWorkspaceFiles: FileData[],
   rustReferences: NxWorkspaceFilesExternals,
   fileMapCache: FileMapCache | null,
-  shouldWriteCache: boolean,
-  plugins: RemotePlugin[]
+  shouldWriteCache: boolean
 ): Promise<{
   projectGraph: ProjectGraph;
   projectFileMapCache: FileMapCache;
@@ -117,8 +118,7 @@ export async function buildProjectGraphUsingProjectFileMap(
     externalNodes,
     context,
     cachedFileData,
-    projectGraphVersion,
-    plugins
+    projectGraphVersion
   );
   const projectFileMapCache = createProjectFileMapCache(
     nxJson,
@@ -129,6 +129,7 @@ export async function buildProjectGraphUsingProjectFileMap(
   if (shouldWriteCache) {
     writeCache(projectFileMapCache, projectGraph);
   }
+  await shutdownPluginWorkers();
   return {
     projectGraph,
     projectFileMapCache,
@@ -164,8 +165,7 @@ async function buildProjectGraphUsingContext(
   knownExternalNodes: Record<string, ProjectGraphExternalNode>,
   ctx: CreateDependenciesContext,
   cachedFileData: CachedFileData,
-  projectGraphVersion: string,
-  plugins: RemotePlugin[]
+  projectGraphVersion: string
 ) {
   performance.mark('build project graph:start');
 
@@ -178,7 +178,7 @@ async function buildProjectGraphUsingContext(
   await normalizeProjectNodes(ctx, builder);
   const initProjectGraph = builder.getUpdatedProjectGraph();
 
-  const r = await updateProjectGraphWithPlugins(ctx, initProjectGraph, plugins);
+  const r = await updateProjectGraphWithPlugins(ctx, initProjectGraph);
 
   const updatedBuilder = new ProjectGraphBuilder(r, ctx.fileMap.projectFileMap);
   for (const proj of Object.keys(cachedFileData.projectFileMap)) {
@@ -235,9 +235,12 @@ function createContext(
 
 async function updateProjectGraphWithPlugins(
   context: CreateDependenciesContext,
-  initProjectGraph: ProjectGraph,
-  plugins: RemotePlugin[]
+  initProjectGraph: ProjectGraph
 ) {
+  const plugins = await loadNxPlugins(
+    context.nxJsonConfiguration?.plugins,
+    context.workspaceRoot
+  );
   let graph = initProjectGraph;
   for (const plugin of plugins) {
     try {