Merge pull request #4674 from g-chao/chao/add-pnpm-sync-config

[rush-lib] Add a config to pnpm-config.json about injected install for cross subspace workspace dependencies

.github/workflows/ci.yml

@@ -55,5 +55,8 @@ jobs:
       - name: Ensure repo README is up-to-date
         run: node repo-scripts/repo-toolbox/lib/start.js readme --verify
 
+      - name: Rush update (rush-lib)
+        run: node apps/rush/lib/start-dev.js update --recheck
+
       - name: Rush test (rush-lib)
         run: node apps/rush/lib/start-dev.js test --verbose --production --timeline

common/changes/@microsoft/rush/chao-add-pnpm-sync-config_2024-05-02-21-19.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@microsoft/rush",
+      "comment": "Add a new setting `alwaysInjectDependenciesFromOtherSubspaces` in pnpm-config.json",
+      "type": "none"
+    }
+  ],
+  "packageName": "@microsoft/rush"
+}

common/config/subspaces/default/repo-state.json

@@ -1,5 +1,5 @@
 // DO NOT MODIFY THIS FILE MANUALLY BUT DO COMMIT IT. It is generated and used by Rush.
 {
-  "pnpmShrinkwrapHash": "e1449a656b11eb9bb4b8a68763cb9727b029a19d",
+  "pnpmShrinkwrapHash": "84413150c512b92dfbc986fab412bca06be42ab1",
   "preferredVersionsHash": "ce857ea0536b894ec8f346aaea08cfd85a5af648"
 }

common/reviews/api/rush-lib.api.md

@@ -682,6 +682,7 @@ export interface IPnpmLockfilePolicies {
 // @internal
 export interface _IPnpmOptionsJson extends IPackageManagerOptionsJsonBase {
     alwaysFullInstall?: boolean;
+    alwaysInjectDependenciesFromOtherSubspaces?: boolean;
     autoInstallPeers?: boolean;
     globalAllowedDeprecatedVersions?: Record<string, string>;
     globalNeverBuiltDependencies?: string[];
@@ -1049,6 +1050,7 @@ export class PhasedCommandHooks {
 // @public
 export class PnpmOptionsConfiguration extends PackageManagerOptionsConfigurationBase {
     readonly alwaysFullInstall: boolean | undefined;
+    readonly alwaysInjectDependenciesFromOtherSubspaces: boolean | undefined;
     readonly autoInstallPeers: boolean | undefined;
     readonly globalAllowedDeprecatedVersions: Record<string, string> | undefined;
     readonly globalNeverBuiltDependencies: string[] | undefined;

libraries/rush-lib/assets/rush-init/common/config/rush/pnpm-config.json

@@ -121,6 +121,28 @@
    */
   /*[LINE "HYPOTHETICAL"]*/ "preventManualShrinkwrapChanges": true,
 
+  /**
+   * When a project uses `workspace:` to depend on another Rush project, PNPM normally installs
+   * it by creating a symlink under `node_modules`.  This generally works well, but in certain
+   * cases such as differing `peerDependencies` versions, symlinking may cause trouble
+   * such as incorrectly satisfied versions.  For such cases, the dependency can be declared
+   * as "injected", causing PNPM to copy its built output into `node_modules` like a real
+   * install from a registry.  Details here: https://rushjs.io/pages/advanced/injected_deps/
+   *
+   * When using Rush subspaces, these sorts of versioning problems are much more likely if
+   * `workspace:` refers to a project from a different subspace.  This is because the symlink
+   * would point to a separate `node_modules` tree installed by a different PNPM lockfile.
+   * A comprehensives solution is to enable `alwaysInjectDependenciesFromOtherSubspaces`,
+   * which automatically treats all projects from other subspaces as injected dependencies
+   * without having to manually configure them.
+   *
+   * NOTE: Use carefully -- excessive file copying can slow down the `rush install` and
+   * `pnpm-sync` operations if too many dependencies become injected.
+   *
+   * The default value is false.
+   */
+  "alwaysInjectDependenciesFromOtherSubspaces": false,
+
   /**
    * Defines the policies to be checked for the `pnpm-lock.yaml` file.
    */

libraries/rush-lib/package.json

@@ -57,7 +57,7 @@
     "tar": "~6.2.1",
     "true-case-path": "~2.2.1",
     "uuid": "~8.3.2",
-    "pnpm-sync-lib": "0.2.2"
+    "pnpm-sync-lib": "0.2.4"
   },
   "devDependencies": {
     "@pnpm/logger": "4.0.0",

libraries/rush-lib/src/cli/RushXCommandLine.ts

@@ -236,7 +236,7 @@ export class RushXCommandLine {
           );
           await pnpmSyncCopyAsync({
             pnpmSyncJsonPath,
-            ensureFolder: FileSystem.ensureFolderAsync,
+            ensureFolderAsync: FileSystem.ensureFolderAsync,
             forEachAsyncWithConcurrency: Async.forEachAsync,
             getPackageIncludedFiles: PackageExtractor.getPackageIncludedFilesAsync,
             logMessageCallback: (logMessageOptions: ILogMessageCallbackOptions) =>

libraries/rush-lib/src/logic/installManager/WorkspaceInstallManager.ts

@@ -525,7 +525,7 @@ export class WorkspaceInstallManager extends BaseInstallManager {
       await pnpmSyncPrepareAsync({
         lockfilePath: pnpmLockfilePath,
         dotPnpmFolder,
-        ensureFolder: FileSystem.ensureFolderAsync,
+        ensureFolderAsync: FileSystem.ensureFolderAsync,
         readPnpmLockfile: async (lockfilePath: string) => {
           const wantedPnpmLockfile: PnpmShrinkwrapFile | undefined = await PnpmShrinkwrapFile.loadFromFile(
             lockfilePath,

libraries/rush-lib/src/logic/operations/PnpmSyncCopyOperationPlugin.ts

@@ -43,7 +43,7 @@ export class PnpmSyncCopyOperationPlugin implements IPhasedCommandPlugin {
             );
             await pnpmSyncCopyAsync({
               pnpmSyncJsonPath,
-              ensureFolder: FileSystem.ensureFolderAsync,
+              ensureFolderAsync: FileSystem.ensureFolderAsync,
               forEachAsyncWithConcurrency: Async.forEachAsync,
               getPackageIncludedFiles: PackageExtractor.getPackageIncludedFilesAsync,
               logMessageCallback: (logMessageOptions: ILogMessageCallbackOptions) =>

libraries/rush-lib/src/logic/pnpm/PnpmOptionsConfiguration.ts

@@ -131,6 +131,10 @@ export interface IPnpmOptionsJson extends IPackageManagerOptionsJsonBase {
    * {@inheritDoc PnpmOptionsConfiguration.autoInstallPeers}
    */
   autoInstallPeers?: boolean;
+  /**
+   * {@inheritDoc PnpmOptionsConfiguration.alwaysInjectDependenciesFromOtherSubspaces}
+   */
+  alwaysInjectDependenciesFromOtherSubspaces?: boolean;
   /**
    * {@inheritDoc PnpmOptionsConfiguration.alwaysFullInstall}
    */
@@ -249,6 +253,18 @@ export class PnpmOptionsConfiguration extends PackageManagerOptionsConfiguration
    */
   public readonly autoInstallPeers: boolean | undefined;
 
+  /**
+   * If true, then `rush update` add injected install options for all cross-subspace
+   * workspace dependencies, to avoid subspace doppelganger issue.
+   *
+   * Here, the injected install refers to PNPM's PNPM's "injected dependencies"
+   * feature. Learn more: https://pnpm.io/package_json#dependenciesmeta
+   *
+   * @remarks
+   * The default value is false.
+   */
+  public readonly alwaysInjectDependenciesFromOtherSubspaces: boolean | undefined;
+
   /**
    * The "globalOverrides" setting provides a simple mechanism for overriding version selections
    * for all dependencies of all projects in the monorepo workspace.  The settings are copied
@@ -389,6 +405,7 @@ export class PnpmOptionsConfiguration extends PackageManagerOptionsConfiguration
     this._globalPatchedDependencies = json.globalPatchedDependencies;
     this.resolutionMode = json.resolutionMode;
     this.autoInstallPeers = json.autoInstallPeers;
+    this.alwaysInjectDependenciesFromOtherSubspaces = json.alwaysInjectDependenciesFromOtherSubspaces;
     this.alwaysFullInstall = json.alwaysFullInstall;
     this.pnpmLockfilePolicies = json.pnpmLockfilePolicies;
   }

libraries/rush-lib/src/logic/pnpm/SubspacePnpmfileConfiguration.ts

@@ -11,6 +11,7 @@ import type { RushConfigurationProject } from '../../api/RushConfigurationProjec
 import type { PnpmPackageManager } from '../../api/packageManager/PnpmPackageManager';
 import { RushConstants } from '../RushConstants';
 import type { Subspace } from '../../api/Subspace';
+import type { PnpmOptionsConfiguration } from './PnpmOptionsConfiguration';
 
 /**
  * Loads PNPM's pnpmfile.js configuration, and invokes it to preprocess package.json files,
@@ -115,11 +116,13 @@ export class SubspacePnpmfileConfiguration {
     const processTransitiveInjectedInstallQueue: Array<RushConfigurationProject> = [];
 
     for (const subspaceProject of subspaceProjectsMap.values()) {
+      const injectedDependencySet: Set<string> = new Set();
       const dependenciesMeta: IDependenciesMetaTable | undefined =
         subspaceProject.packageJson.dependenciesMeta;
       if (dependenciesMeta) {
         for (const [dependencyName, { injected }] of Object.entries(dependenciesMeta)) {
           if (injected) {
+            injectedDependencySet.add(dependencyName);
             projectNameToInjectedDependenciesMap.get(subspaceProject.packageName)?.add(dependencyName);
 
             //if this dependency is in the same subspace, leave as it is, PNPM will handle it
@@ -131,6 +134,24 @@ export class SubspacePnpmfileConfiguration {
           }
         }
       }
+
+      // if alwaysInjectDependenciesFromOtherSubspaces policy is true in pnpm-config.json
+      // and the dependency is not injected yet
+      // and the dependency is in another subspace
+      // then, make this dependency as injected dependency
+      const pnpmOptions: PnpmOptionsConfiguration | undefined =
+        subspace.getPnpmOptions() || rushConfiguration.pnpmOptions;
+      if (pnpmOptions && pnpmOptions.alwaysInjectDependenciesFromOtherSubspaces) {
+        const dependencyProjects: ReadonlySet<RushConfigurationProject> = subspaceProject.dependencyProjects;
+        for (const dependencyProject of dependencyProjects) {
+          const dependencyName: string = dependencyProject.packageName;
+          if (!injectedDependencySet.has(dependencyName) && !subspaceProjectsMap.has(dependencyName)) {
+            projectNameToInjectedDependenciesMap.get(subspaceProject.packageName)?.add(dependencyName);
+            // process transitive injected installation
+            processTransitiveInjectedInstallQueue.push(workspaceProjectsMap.get(dependencyName)!);
+          }
+        }
+      }
     }
 
     // rewrite all workspace dependencies to injected install all for transitive injected installation case

libraries/rush-lib/src/schemas/pnpm-config.schema.json

@@ -48,6 +48,11 @@
       "type": "boolean"
     },
 
+    "alwaysInjectDependenciesFromOtherSubspaces": {
+      "description": "When a project uses `workspace:` to depend on another Rush project, PNPM normally installs it by creating a symlink under `node_modules`.  This generally works well, but in certain cases such as differing `peerDependencies` versions, symlinking may cause trouble such as incorrectly satisfied versions.  For such cases, the dependency can be declared as \"injected\", causing PNPM to copy its built output into `node_modules` like a real install from a registry.  Details here: https://rushjs.io/pages/advanced/injected_deps/\n\nWhen using Rush subspaces, these sorts of versioning problems are much more likely if `workspace:` refers to a project from a different subspace.  This is because the symlink would point to a separate `node_modules` tree installed by a different PNPM lockfile.  A comprehensives solution is to enable `alwaysInjectDependenciesFromOtherSubspaces`, which automatically treats all projects from other subspaces as injected dependencies without having to manually configure them.\n\nNOTE: Use carefully -- excessive file copying can slow down the `rush install` and `pnpm-sync` operations if too many dependencies become injected.\n\nThe default value is false.",
+      "type": "boolean"
+    },
+
     "globalOverrides": {
       "description": "The \"globalOverrides\" setting provides a simple mechanism for overriding version selections for all dependencies of all projects in the monorepo workspace. The settings are copied into the `pnpm.overrides` field of the `common/temp/package.json` file that is generated by Rush during installation.\n\nOrder of precedence: `.pnpmfile.cjs` has the highest precedence, followed by `unsupportedPackageJsonSettings`, `globalPeerDependencyRules`, `globalPackageExtensions`, and `globalOverrides` has lowest precedence.\n\nPNPM documentation: https://pnpm.io/package_json#pnpmoverrides",
       "type": "object",

rush.json

@@ -16,7 +16,7 @@
    * path segment in the "$schema" field for all your Rush config files.  This will ensure
    * correct error-underlining and tab-completion for editors such as VS Code.
    */
-  "rushVersion": "5.120.2",
+  "rushVersion": "5.122.0",
 
   /**
    * The next field selects which package manager should be installed and determines its version.