feat(core): ability to run local nx plugins w/o build step (#9116)

* feat(core): ability to run local nx plugins generators and executors

* feat(core): support for project inference and graph extension local plugins

* chore(core): cleanup after package relocation

* chore(core): changes for review

e2e/nx-plugin/src/nx-plugin.test.ts

@@ -6,15 +6,20 @@ import {
   newProject,
   readJson,
   readProjectConfig,
-  readWorkspaceConfig,
   runCLI,
   runCLIAsync,
   uniq,
-  workspaceConfigName,
+  updateFile,
+  createFile,
+  readFile,
+  removeFile,
 } from '@nrwl/e2e/utils';
 
 describe('Nx Plugin', () => {
-  beforeEach(() => newProject());
+  let npmScope: string;
+  beforeEach(() => {
+    npmScope = newProject();
+  });
 
   it('should be able to generate a Nx Plugin ', async () => {
     const plugin = uniq('plugin');
@@ -172,6 +177,75 @@ describe('Nx Plugin', () => {
     });
   }, 90000);
 
+  describe('local plugins', () => {
+    const plugin = uniq('plugin');
+    beforeEach(() => {
+      runCLI(`generate @nrwl/nx-plugin:plugin ${plugin} --linter=eslint`);
+    });
+
+    it('should be able to infer projects and targets', async () => {
+      // Cache workspace json, to test inference and restore afterwards
+      const workspaceJsonContents = readFile('workspace.json');
+      removeFile('workspace.json');
+
+      // Setup project inference + target inference
+      updateFile(
+        `libs/${plugin}/src/index.ts`,
+        `import {basename} from 'path'
+  
+  export function registerProjectTargets(f) {
+    if (basename(f) === 'my-project-file') {
+      return {
+        build: {
+          executor: "@nrwl/workspace:run-commands",
+          options: {
+            command: "echo 'custom registered target'"
+          }
+        }
+      }
+    }
+  }
+  
+  export const projectFilePatterns = ['my-project-file'];
+  `
+      );
+
+      // Register plugin in nx.json (required for inference)
+      updateFile(`nx.json`, (nxJson) => {
+        const nx = JSON.parse(nxJson);
+        nx.plugins = [`@${npmScope}/${plugin}`];
+        return JSON.stringify(nx, null, 2);
+      });
+
+      // Create project that should be inferred by Nx
+      const inferredProject = uniq('inferred');
+      createFile(`libs/${inferredProject}/my-project-file`);
+
+      // Attempt to use inferred project w/ Nx
+      expect(runCLI(`build ${inferredProject}`)).toContain(
+        'custom registered target'
+      );
+
+      // Restore workspace.json
+      createFile('workspace.json', workspaceJsonContents);
+    });
+
+    it('should be able to use local generators and executors', async () => {
+      const generator = uniq('generator');
+      const generatedProject = uniq('project');
+
+      runCLI(
+        `generate @nrwl/nx-plugin:generator ${generator} --project=${plugin}`
+      );
+
+      runCLI(
+        `generate @${npmScope}/${plugin}:${generator} --name ${generatedProject}`
+      );
+      expect(() => checkFilesExist(`libs/${generatedProject}`)).not.toThrow();
+      expect(() => runCLI(`build ${generatedProject}`)).not.toThrow();
+    });
+  });
+
   describe('--directory', () => {
     it('should create a plugin in the specified directory', () => {
       const plugin = uniq('plugin');

package.json

@@ -89,7 +89,7 @@
     "@storybook/core": "~6.4.12",
     "@storybook/react": "~6.4.12",
     "@svgr/webpack": "^6.1.2",
-    "@swc/core": "^1.2.146",
+    "@swc/core": "^1.2.152",
     "@swc-node/register": "^1.4.2",
     "@testing-library/react": "11.2.6",
     "@testing-library/react-hooks": "7.0.1",

packages/nx/package.json

@@ -28,7 +28,7 @@
   },
   "homepage": "https://nx.dev",
   "dependencies": {
-    "@swc/core": "^1.2.146",
+    "@swc/core": "^1.2.152",
     "@swc-node/register": "^1.4.2",
     "chalk": "4.1.0",
     "enquirer": "~2.3.6",

packages/nx/src/shared/nx-plugin.ts

@@ -1,10 +1,18 @@
 import { sync } from 'fast-glob';
 import { existsSync } from 'fs';
-import { dirname, join } from 'path';
+import * as path from 'path';
+
 import { appRootPath } from '../utils/app-root';
 import { readJsonFile } from '../utils/fileutils';
+import { registerTsProject } from '../utils/register';
+import { PackageJson } from './package-json';
 import { ProjectGraphProcessor } from './project-graph';
-import { TargetConfiguration } from './workspace';
+import { Workspaces } from './workspace';
+import {
+  ProjectConfiguration,
+  TargetConfiguration,
+  WorkspaceJsonConfiguration,
+} from './workspace.model';
 
 export type ProjectTargetConfigurator = (
   file: string
@@ -25,30 +33,40 @@ export interface NxPlugin {
   projectFilePatterns?: string[];
 }
 
-function findPluginPackageJson(path: string, plugin: string) {
-  while (true) {
-    if (!path.startsWith(appRootPath)) {
-      throw new Error("Couldn't find a package.json for Nx plugin:" + plugin);
-    }
-    if (existsSync(join(path, 'package.json'))) {
-      return join(path, 'package.json');
-    }
-    path = dirname(path);
-  }
-}
-
+// Short lived cache (cleared between cmd runs)
+// holding resolved nx plugin objects.
+// Allows loadNxPlugins to be called multiple times w/o
+// executing resolution mulitple times.
 let nxPluginCache: NxPlugin[] = null;
-export function loadNxPlugins(plugins?: string[]): NxPlugin[] {
+export function loadNxPlugins(
+  plugins?: string[],
+  paths = [appRootPath]
+): NxPlugin[] {
   return plugins?.length
     ? nxPluginCache ||
-        (nxPluginCache = plugins.map((path) => {
-          const pluginPath = require.resolve(path, {
-            paths: [appRootPath],
-          });
-
-          const { name } = readJsonFile(
-            findPluginPackageJson(pluginPath, path)
-          );
+        (nxPluginCache = plugins.map((moduleName) => {
+          let pluginPath: string;
+          try {
+            pluginPath = require.resolve(moduleName, {
+              paths,
+            });
+          } catch (e) {
+            if (e.code === 'MODULE_NOT_FOUND') {
+              const plugin = resolveLocalNxPlugin(moduleName);
+              const main = readPluginMainFromProjectConfiguration(
+                plugin.projectConfig
+              );
+              pluginPath = main ? path.join(appRootPath, main) : plugin.path;
+            } else {
+              throw e;
+            }
+          }
+          const packageJsonPath = path.join(pluginPath, 'package.json');
+          const { name } =
+            !['.ts', '.js'].some((x) => x === path.extname(pluginPath)) && // Not trying to point to a ts or js file
+            existsSync(packageJsonPath) // plugin has a package.json
+              ? readJsonFile(packageJsonPath) // read name from package.json
+              : { name: path.basename(pluginPath) }; // use the name of the file we point to
           const plugin = require(pluginPath) as NxPlugin;
           plugin.name = name;
 
@@ -69,14 +87,151 @@ export function mergePluginTargetsWithNxTargets(
     }
 
     const projectFiles = sync(`+(${plugin.projectFilePatterns.join('|')})`, {
-      cwd: join(appRootPath, projectRoot),
+      cwd: path.join(appRootPath, projectRoot),
     });
     for (const projectFile of projectFiles) {
       newTargets = {
         ...newTargets,
-        ...plugin.registerProjectTargets(join(projectRoot, projectFile)),
+        ...plugin.registerProjectTargets(path.join(projectRoot, projectFile)),
       };
     }
   }
   return { ...newTargets, ...targets };
 }
+
+export function readPluginPackageJson(
+  pluginName: string,
+  paths = [appRootPath]
+): {
+  path: string;
+  json: PackageJson;
+} {
+  let packageJsonPath: string;
+  try {
+    packageJsonPath = require.resolve(`${pluginName}/package.json`, {
+      paths,
+    });
+  } catch (e) {
+    if (e.code === 'MODULE_NOT_FOUND') {
+      const localPluginPath = resolveLocalNxPlugin(pluginName);
+      if (localPluginPath) {
+        const localPluginPackageJson = path.join(
+          localPluginPath.path,
+          'package.json'
+        );
+        return {
+          path: localPluginPackageJson,
+          json: readJsonFile(localPluginPackageJson),
+        };
+      }
+    }
+    throw e;
+  }
+  return { json: readJsonFile(packageJsonPath), path: packageJsonPath };
+}
+
+/**
+ * Builds a plugin package and returns the path to output
+ * @param importPath What is the import path that refers to a potential plugin?
+ * @returns The path to the built plugin, or null if it doesn't exist
+ */
+const localPluginCache: Record<
+  string,
+  { path: string; projectConfig: ProjectConfiguration }
+> = {};
+export function resolveLocalNxPlugin(
+  importPath: string,
+  root = appRootPath
+): { path: string; projectConfig: ProjectConfiguration } | null {
+  localPluginCache[importPath] ??= lookupLocalPlugin(importPath, root);
+  return localPluginCache[importPath];
+}
+
+let tsNodeAndPathsRegistered = false;
+function registerTSTranspiler() {
+  if (!tsNodeAndPathsRegistered) {
+    registerTsProject(appRootPath, 'tsconfig.base.json');
+  }
+  tsNodeAndPathsRegistered = true;
+}
+
+function lookupLocalPlugin(importPath: string, root = appRootPath) {
+  const workspace = new Workspaces(root).readWorkspaceConfiguration({
+    _ignorePluginInference: true,
+  });
+  const plugin = findNxProjectForImportPath(importPath, workspace, root);
+  if (!plugin) {
+    return null;
+  }
+
+  if (!tsNodeAndPathsRegistered) {
+    registerTSTranspiler();
+  }
+
+  const projectConfig = workspace.projects[plugin];
+  return { path: path.join(root, projectConfig.root), projectConfig };
+}
+
+function findNxProjectForImportPath(
+  importPath: string,
+  workspace: WorkspaceJsonConfiguration,
+  root = appRootPath
+): string | null {
+  const tsConfigPaths: Record<string, string[]> = readTsConfigPaths(root);
+  const possiblePaths = tsConfigPaths[importPath]?.map((p) =>
+    path.resolve(root, p)
+  );
+  if (tsConfigPaths[importPath]) {
+    const projectRootMappings = Object.entries(workspace.projects).reduce(
+      (m, [project, config]) => {
+        m[path.resolve(root, config.root)] = project;
+        return m;
+      },
+      {}
+    );
+    for (const root of Object.keys(projectRootMappings)) {
+      if (possiblePaths.some((p) => p.startsWith(root))) {
+        return projectRootMappings[root];
+      }
+    }
+    if (process.env.NX_VERBOSE_LOGGING) {
+      console.log(
+        'Unable to find local plugin',
+        possiblePaths,
+        projectRootMappings
+      );
+    }
+    throw new Error(
+      'Unable to resolve local plugin with import path ' + importPath
+    );
+  }
+}
+
+let tsconfigPaths: Record<string, string[]>;
+function readTsConfigPaths(root: string = appRootPath) {
+  if (!tsconfigPaths) {
+    const tsconfigPath: string | null = ['tsconfig.base.json', 'tsconfig.json']
+      .map((x) => path.join(root, x))
+      .filter((x) => existsSync(x))[0];
+    if (!tsconfigPath) {
+      throw new Error('unable to find tsconfig.base.json or tsconfig.json');
+    }
+    const { compilerOptions } = readJsonFile(tsconfigPath);
+    tsconfigPaths = compilerOptions?.paths;
+  }
+  return tsconfigPaths;
+}
+
+function readPluginMainFromProjectConfiguration(
+  plugin: ProjectConfiguration
+): string | null {
+  const { main } =
+    Object.values(plugin.targets).find((x) =>
+      ['@nrwl/js:tsc', '@nrwl/js:swc', '@nrwl/node:package'].includes(
+        x.executor
+      )
+    )?.options ||
+    plugin.targets?.build?.options ||
+    {};
+  return main;
+}