fix: rebuild when node-linker is set to hoisted (#5811)

close #5560

.changeset/afraid-gifts-provide.md

@@ -0,0 +1,6 @@
+---
+"@pnpm/plugin-commands-rebuild": patch
+"pnpm": patch
+---
+
+`pnpm rebuild` should work in projects that use the hoister node linker [#5560](https://github.com/pnpm/pnpm/issues/5560).

.changeset/rare-rocks-type.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/fs.hard-link-dir": major
+---
+
+Initial release.

__utils__/assert-project/package.json

@@ -44,7 +44,7 @@
     "@pnpm/constants": "workspace:*",
     "@pnpm/lockfile-types": "workspace:*",
     "@pnpm/modules-yaml": "workspace:*",
-    "@pnpm/registry-mock": "3.1.0",
+    "@pnpm/registry-mock": "3.2.0",
     "@pnpm/types": "workspace:*",
     "is-windows": "^1.0.2",
     "isexe": "2.0.0",

__utils__/assert-store/package.json

@@ -41,7 +41,7 @@
   },
   "dependencies": {
     "@pnpm/cafs": "workspace:*",
-    "@pnpm/registry-mock": "3.1.0",
+    "@pnpm/registry-mock": "3.2.0",
     "path-exists": "^4.0.0"
   },
   "devDependencies": {

exec/plugin-commands-rebuild/package.json

@@ -33,10 +33,11 @@
   },
   "homepage": "https://github.com/pnpm/pnpm/blob/main/exec/plugin-commands-rebuild#readme",
   "devDependencies": {
+    "@pnpm/assert-project": "workspace:*",
     "@pnpm/filter-workspace-packages": "workspace:*",
     "@pnpm/plugin-commands-rebuild": "workspace:*",
     "@pnpm/prepare": "workspace:*",
-    "@pnpm/registry-mock": "3.1.0",
+    "@pnpm/registry-mock": "3.2.0",
     "@pnpm/test-fixtures": "workspace:*",
     "@types/ramda": "0.28.20",
     "@types/semver": "7.3.13",
@@ -52,7 +53,9 @@
     "@pnpm/config": "workspace:*",
     "@pnpm/constants": "workspace:*",
     "@pnpm/core-loggers": "workspace:*",
+    "@pnpm/dependency-path": "workspace:*",
     "@pnpm/find-workspace-packages": "workspace:*",
+    "@pnpm/fs.hard-link-dir": "workspace:*",
     "@pnpm/get-context": "workspace:*",
     "@pnpm/graph-sequencer": "1.0.0",
     "@pnpm/lifecycle": "workspace:*",
@@ -66,7 +69,6 @@
     "@pnpm/store-connection-manager": "workspace:*",
     "@pnpm/store-controller-types": "workspace:*",
     "@pnpm/types": "workspace:*",
-    "@pnpm/dependency-path": "workspace:*",
     "load-json-file": "^6.2.0",
     "mem": "^8.1.1",
     "p-limit": "^3.1.0",

exec/plugin-commands-rebuild/src/implementation/index.ts

@@ -4,7 +4,7 @@ import {
   WANTED_LOCKFILE,
 } from '@pnpm/constants'
 import { skippedOptionalDependencyLogger } from '@pnpm/core-loggers'
-import { getContext } from '@pnpm/get-context'
+import { getContext, PnpmContext } from '@pnpm/get-context'
 import {
   runLifecycleHooksConcurrently,
   runPostinstallHooks,
@@ -22,6 +22,7 @@ import { writeModulesManifest } from '@pnpm/modules-yaml'
 import { createOrConnectStoreController } from '@pnpm/store-connection-manager'
 import { ProjectManifest } from '@pnpm/types'
 import * as dp from '@pnpm/dependency-path'
+import { hardLinkDir } from '@pnpm/fs.hard-link-dir'
 import runGroups from 'run-groups'
 import graphSequencer from '@pnpm/graph-sequencer'
 import npa from '@pnpm/npm-package-arg'
@@ -231,7 +232,7 @@ async function _rebuild (
     projects: Record<string, { id: string, rootDir: string }>
     extraBinPaths: string[]
     extraNodePaths: string[]
-  },
+  } & Pick<PnpmContext, 'modulesFile'>,
   opts: StrictRebuildOptions
 ) {
   const pkgsThatWereRebuilt = new Set()
@@ -272,14 +273,22 @@ async function _rebuild (
     async () => {
       const pkgSnapshot = pkgSnapshots[depPath]
       const pkgInfo = nameVerFromPkgSnapshot(depPath, pkgSnapshot)
-      const pkgRoot = path.join(ctx.virtualStoreDir, dp.depPathToFilename(depPath), 'node_modules', pkgInfo.name)
+      const pkgRoots = opts.nodeLinker === 'hoisted'
+        ? (ctx.modulesFile?.hoistedLocations?.[depPath] ?? []).map((hoistedLocation) => path.join(opts.lockfileDir, hoistedLocation))
+        : [path.join(ctx.virtualStoreDir, dp.depPathToFilename(depPath), 'node_modules', pkgInfo.name)]
+      const pkgRoot = pkgRoots[0]
       try {
-        const modules = path.join(ctx.virtualStoreDir, dp.depPathToFilename(depPath), 'node_modules')
-        const binPath = path.join(pkgRoot, 'node_modules', '.bin')
-        await linkBins(modules, binPath, { extraNodePaths: ctx.extraNodePaths, warn })
+        const extraBinPaths = ctx.extraBinPaths
+        if (opts.nodeLinker !== 'hoisted') {
+          const modules = path.join(ctx.virtualStoreDir, dp.depPathToFilename(depPath), 'node_modules')
+          const binPath = path.join(pkgRoot, 'node_modules', '.bin')
+          await linkBins(modules, binPath, { extraNodePaths: ctx.extraNodePaths, warn })
+        } else {
+          extraBinPaths.push(...binDirsInAllParentDirs(pkgRoot, opts.lockfileDir))
+        }
         await runPostinstallHooks({
           depPath,
-          extraBinPaths: ctx.extraBinPaths,
+          extraBinPaths,
           extraEnv: opts.extraEnv,
           optional: pkgSnapshot.optional === true,
           pkgRoot,
@@ -307,6 +316,9 @@ async function _rebuild (
         }
         throw err
       }
+      if (pkgRoots.length > 1) {
+        await hardLinkDir(pkgRoot, pkgRoots.slice(1))
+      }
     }
   ))
 
@@ -336,3 +348,16 @@ async function _rebuild (
 
   return pkgsThatWereRebuilt
 }
+
+function binDirsInAllParentDirs (pkgRoot: string, lockfileDir: string): string[] {
+  const binDirs: string[] = []
+  let dir = pkgRoot
+  do {
+    if (!path.dirname(dir).startsWith('@')) {
+      binDirs.push(path.join(dir, 'node_modules/.bin'))
+    }
+    dir = path.dirname(dir)
+  } while (path.relative(dir, lockfileDir) !== '')
+  binDirs.push(path.join(lockfileDir, 'node_modules/.bin'))
+  return binDirs
+}

exec/plugin-commands-rebuild/src/rebuild.ts

@@ -76,6 +76,8 @@ export async function handler (
   | 'dir'
   | 'engineStrict'
   | 'hooks'
+  | 'lockfileDir'
+  | 'nodeLinker'
   | 'rawLocalConfig'
   | 'registries'
   | 'scriptShell'

exec/plugin-commands-rebuild/src/recursive.ts

@@ -22,6 +22,7 @@ type RecursiveRebuildOpts = CreateStoreControllerOptions & Pick<Config,
 | 'ignoreScripts'
 | 'lockfileDir'
 | 'lockfileOnly'
+| 'nodeLinker'
 | 'rawLocalConfig'
 | 'registries'
 | 'sharedWorkspaceLockfile'

exec/plugin-commands-rebuild/test/index.ts

@@ -23,7 +23,7 @@ test('rebuilds dependencies', async () => {
     pnpmBin,
     'add',
     '--save-dev',
-    '@pnpm.e2e/pre-and-postinstall-scripts-example',
+    '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0',
     'pnpm/test-git-fetch#299c6d89507571462b992b92407a8a07663e32ee',
     `--registry=${REGISTRY}`,
     `--store-dir=${storeDir}`,
@@ -109,7 +109,7 @@ test('rebuilds specific dependencies', async () => {
     pnpmBin,
     'add',
     '--save-dev',
-    '@pnpm.e2e/pre-and-postinstall-scripts-example',
+    '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0',
     'pnpm-e2e/install-scripts-example#b6cfdb8af6f8d5ebc5e7de6831af9d38084d765b',
     `--registry=${REGISTRY}`,
     `--store-dir=${storeDir}`,
@@ -144,7 +144,7 @@ test('rebuild with pending option', async () => {
   await execa('node', [
     pnpmBin,
     'add',
-    '@pnpm.e2e/pre-and-postinstall-scripts-example',
+    '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0',
     `--registry=${REGISTRY}`,
     `--store-dir=${storeDir}`,
     `--cache-dir=${cacheDir}`,

exec/plugin-commands-rebuild/test/recursive.ts

@@ -1,4 +1,5 @@
 import path from 'path'
+import { assertProject } from '@pnpm/assert-project'
 import { readProjects } from '@pnpm/filter-workspace-packages'
 import { rebuild } from '@pnpm/plugin-commands-rebuild'
 import { preparePackages } from '@pnpm/prepare'
@@ -63,6 +64,89 @@ test('pnpm recursive rebuild', async () => {
   await projects['project-2'].has('@pnpm.e2e/pre-and-postinstall-scripts-example/generated-by-postinstall.js')
 })
 
+test('pnpm recursive rebuild with hoisted node linker', async () => {
+  const projects = preparePackages([
+    {
+      name: 'project-1',
+      version: '1.0.0',
+
+      dependencies: {
+        '@pnpm.e2e/pre-and-postinstall-scripts-example': '1',
+      },
+    },
+    {
+      name: 'project-2',
+      version: '1.0.0',
+
+      dependencies: {
+        '@pnpm.e2e/pre-and-postinstall-scripts-example': '1',
+      },
+    },
+    {
+      name: 'project-3',
+      version: '1.0.0',
+
+      dependencies: {
+        '@pnpm.e2e/pre-and-postinstall-scripts-example': '2',
+      },
+    },
+    {
+      name: 'project-4',
+      version: '1.0.0',
+
+      dependencies: {
+        '@pnpm.e2e/pre-and-postinstall-scripts-example': '2',
+      },
+    },
+  ])
+
+  const { allProjects, selectedProjectsGraph } = await readProjects(process.cwd(), [])
+  await writeYamlFile('pnpm-workspace.yaml', { packages: ['*'] })
+  await execa('node', [
+    pnpmBin,
+    'install',
+    '-r',
+    `--registry=${REGISTRY}`,
+    `--store-dir=${path.resolve(DEFAULT_OPTS.storeDir)}`,
+    `--cache-dir=${path.resolve(DEFAULT_OPTS.cacheDir)}`,
+    '--ignore-scripts',
+    '--reporter=append-only',
+    '--config.node-linker=hoisted',
+  ], { stdout: 'inherit' })
+
+  const rootProject = assertProject(process.cwd())
+  await rootProject.hasNot('@pnpm.e2e/pre-and-postinstall-scripts-example/generated-by-preinstall.js')
+  await rootProject.hasNot('@pnpm.e2e/pre-and-postinstall-scripts-example/generated-by-postinstall.js')
+  await projects['project-3'].hasNot('@pnpm.e2e/pre-and-postinstall-scripts-example/generated-by-preinstall.js')
+  await projects['project-3'].hasNot('@pnpm.e2e/pre-and-postinstall-scripts-example/generated-by-postinstall.js')
+  await projects['project-4'].hasNot('@pnpm.e2e/pre-and-postinstall-scripts-example/generated-by-preinstall.js')
+  await projects['project-4'].hasNot('@pnpm.e2e/pre-and-postinstall-scripts-example/generated-by-postinstall.js')
+
+  const modulesManifest = await rootProject.readModulesManifest()
+  await rebuild.handler({
+    ...DEFAULT_OPTS,
+    allProjects,
+    dir: process.cwd(),
+    nodeLinker: 'hoisted',
+    recursive: true,
+    registries: modulesManifest!.registries!,
+    selectedProjectsGraph,
+    lockfileDir: process.cwd(),
+    workspaceDir: process.cwd(),
+  }, [])
+
+  await rootProject.has('@pnpm.e2e/pre-and-postinstall-scripts-example/generated-by-preinstall.js')
+  await rootProject.has('@pnpm.e2e/pre-and-postinstall-scripts-example/generated-by-postinstall.js')
+  await projects['project-1'].hasNot('@pnpm.e2e/pre-and-postinstall-scripts-example/generated-by-preinstall.js')
+  await projects['project-1'].hasNot('@pnpm.e2e/pre-and-postinstall-scripts-example/generated-by-postinstall.js')
+  await projects['project-2'].hasNot('@pnpm.e2e/pre-and-postinstall-scripts-example/generated-by-preinstall.js')
+  await projects['project-2'].hasNot('@pnpm.e2e/pre-and-postinstall-scripts-example/generated-by-postinstall.js')
+  await projects['project-3'].has('@pnpm.e2e/pre-and-postinstall-scripts-example/generated-by-preinstall.js')
+  await projects['project-3'].has('@pnpm.e2e/pre-and-postinstall-scripts-example/generated-by-postinstall.js')
+  await projects['project-4'].has('@pnpm.e2e/pre-and-postinstall-scripts-example/generated-by-preinstall.js')
+  await projects['project-4'].has('@pnpm.e2e/pre-and-postinstall-scripts-example/generated-by-postinstall.js')
+})
+
 // TODO: make this test pass
 test.skip('rebuild multiple packages in correct order', async () => {
   const pkgs = [

exec/plugin-commands-rebuild/tsconfig.json

@@ -9,6 +9,9 @@
     "../../__typings__/**/*.d.ts"
   ],
   "references": [
+    {
+      "path": "../../__utils__/assert-project"
+    },
     {
       "path": "../../__utils__/prepare"
     },
@@ -27,6 +30,9 @@
     {
       "path": "../../config/normalize-registries"
     },
+    {
+      "path": "../../fs/hard-link-dir"
+    },
     {
       "path": "../../lockfile/lockfile-utils"
     },

exec/plugin-commands-script-runners/package.json

@@ -37,7 +37,7 @@
     "@pnpm/filter-workspace-packages": "workspace:*",
     "@pnpm/plugin-commands-script-runners": "workspace:*",
     "@pnpm/prepare": "workspace:*",
-    "@pnpm/registry-mock": "3.1.0",
+    "@pnpm/registry-mock": "3.2.0",
     "@types/is-windows": "^1.0.0",
     "@types/ramda": "0.28.20",
     "is-windows": "^1.0.2",

fs/hard-link-dir/README.md

@@ -0,0 +1,17 @@
+# @pnpm/fs.hard-link-dir
+
+> Hard link all files from a directory to several target directories.
+
+<!--@shields('npm')-->
+[![npm version](https://img.shields.io/npm/v/hard-link-dir.svg)](https://www.npmjs.com/package/@pnpm/fs.hard-link-dir)
+<!--/@-->
+
+## Installation
+
+```sh
+pnpm add @pnpm/fs.hard-link-dir
+```
+
+## License
+
+MIT © [Zoltan Kochan](https://www.kochan.io)

fs/hard-link-dir/jest.config.js

@@ -0,0 +1 @@
+module.exports = require('../../jest.config.js')

fs/hard-link-dir/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@pnpm/fs.hard-link-dir",
+  "version": "0.0.0",
+  "description": "Hard link all files from a directory to several target directories.",
+  "main": "lib/index.js",
+  "files": [
+    "lib",
+    "!*.map"
+  ],
+  "types": "lib/index.d.ts",
+  "scripts": {
+    "lint": "eslint src/**/*.ts test/**/*.ts",
+    "_test": "jest",
+    "test": "pnpm run compile && pnpm run _test",
+    "prepublishOnly": "pnpm run compile",
+    "compile": "tsc --build && pnpm run lint --fix"
+  },
+  "repository": "https://github.com/pnpm/pnpm/blob/main/fs/hard-link-dir",
+  "keywords": [
+    "pnpm7",
+    "find",
+    "package"
+  ],
+  "engines": {
+    "node": ">=14.6"
+  },
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/pnpm/pnpm/issues"
+  },
+  "homepage": "https://github.com/pnpm/pnpm/blob/main/fs/hard-link-dir#readme",
+  "dependencies": {
+  },
+  "funding": "https://opencollective.com/pnpm",
+  "devDependencies": {
+    "@pnpm/fs.hard-link-dir": "workspace:*",
+    "@pnpm/prepare": "workspace:*"
+  },
+  "exports": {
+    ".": "./lib/index.js"
+  }
+}