fix: built packages should not modify the original files in the store

.changeset/good-taxis-laugh.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/build-modules": patch
+---
+
+`requiresBuild` may be of any value. This is just a workaround to a typing issue. `requiresBuild` will always be boolean.

.changeset/hungry-dogs-draw.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/resolve-dependencies": major
+---
+
+`requiresBuild` is sometimes a function that return a boolean promise.

.changeset/seven-cats-lie.md

@@ -0,0 +1,7 @@
+---
+"@pnpm/core": patch
+"@pnpm/headless": patch
+"pnpm": patch
+---
+
+Packages that should be built are always cloned or copied from the store. This is required to prevent the postinstall scripts from modifying the original source files of the package.

.changeset/sour-stingrays-tell.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/package-requester": patch
+---
+
+Use `safe-promise-defer`.

.changeset/stale-cheetahs-warn.md

@@ -0,0 +1,6 @@
+---
+"@pnpm/create-cafs-store": minor
+"@pnpm/fetcher-base": minor
+---
+
+New optional option added to package importer: `requiresBuild`. When `requiresBuild` is `true`, the package should only be imported using cloning or copying.

.changeset/tiny-ravens-brake.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/create-cafs-store": minor
+---
+
+New import method added: `clone-or-copy`.

package.json

@@ -39,7 +39,7 @@
     "@commitlint/prompt-cli": "^16.0.0",
     "@pnpm/eslint-config": "workspace:*",
     "@pnpm/meta-updater": "0.0.6",
-    "@pnpm/registry-mock": "2.19.0",
+    "@pnpm/registry-mock": "2.20.0",
     "@pnpm/tsconfig": "workspace:*",
     "@types/jest": "^27.4.0",
     "@types/node": "^14.17.32",

packages/build-modules/src/buildSequence.ts

@@ -14,7 +14,8 @@ export interface DependenciesGraphNode {
   isBuilt?: boolean
   optional: boolean
   optionalDependencies: Set<string>
-  requiresBuild?: boolean
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  requiresBuild?: boolean | any // this is a durty workaround added in https://github.com/pnpm/pnpm/pull/4898
 }
 
 export interface DependenciesGraph {

packages/core/package.json

@@ -76,7 +76,7 @@
     "@pnpm/logger": "^4.0.0",
     "@pnpm/package-store": "workspace:13.0.7",
     "@pnpm/prepare": "workspace:*",
-    "@pnpm/registry-mock": "2.19.0",
+    "@pnpm/registry-mock": "2.20.0",
     "@pnpm/store-path": "workspace:6.0.0",
     "@pnpm/test-fixtures": "workspace:*",
     "@types/fs-extra": "^9.0.5",

packages/core/src/install/index.ts

@@ -843,8 +843,13 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => {
       // we can use concat here because we always only append new packages, which are guaranteed to not be there by definition
       ctx.pendingBuilds = ctx.pendingBuilds
         .concat(
-          result.newDepPaths
-            .filter((depPath) => dependenciesGraph[depPath].requiresBuild)
+          await pFilter(result.newDepPaths,
+            (depPath) => {
+              const requiresBuild = dependenciesGraph[depPath].requiresBuild
+              if (typeof requiresBuild === 'function') return requiresBuild()
+              return requiresBuild
+            }
+          )
         )
     } else if (result.newDepPaths?.length) {
       // postinstall hooks

packages/core/src/install/link.ts

@@ -408,10 +408,14 @@ async function linkAllPkgs (
       if (opts.sideEffectsCacheRead && filesResponse.sideEffects && !isEmpty(filesResponse.sideEffects)) {
         targetEngine = calcDepState(depNode.depPath, opts.depGraph, opts.depsStateCache)
       }
+      if (typeof depNode.requiresBuild === 'function') {
+        depNode.requiresBuild = await depNode.requiresBuild()
+      }
       const { importMethod, isBuilt } = await storeController.importPackage(depNode.dir, {
         filesResponse,
         force: opts.force,
         targetEngine,
+        requiresBuild: depNode.requiresBuild,
       })
       if (importMethod) {
         progressLogger.debug({

packages/core/test/install/sideEffects.ts

@@ -1,8 +1,8 @@
-import { promises as fs } from 'fs'
+import { promises as fs, readFileSync } from 'fs'
 import path from 'path'
 import { addDependenciesToPackage } from '@pnpm/core'
-import { PackageFilesIndex } from '@pnpm/cafs'
-import { REGISTRY_MOCK_PORT } from '@pnpm/registry-mock'
+import { getFilePathInCafs, PackageFilesIndex } from '@pnpm/cafs'
+import { getIntegrity, REGISTRY_MOCK_PORT } from '@pnpm/registry-mock'
 import { prepareEmpty } from '@pnpm/prepare'
 import { ENGINE_NAME } from '@pnpm/constants'
 import rimraf from '@zkochan/rimraf'
@@ -159,3 +159,28 @@ test('uploading errors do not interrupt installation', async () => {
   const filesIndex = await loadJsonFile<PackageFilesIndex>(filesIndexFile)
   expect(filesIndex.sideEffects).toBeFalsy()
 })
+
+test('a postinstall script does not modify the original sources added to the store', async () => {
+  prepareEmpty()
+
+  const opts = await testDefaults({
+    fastUnpack: false,
+    sideEffectsCacheRead: true,
+    sideEffectsCacheWrite: true,
+  }, {}, {}, { packageImportMethod: 'hardlink' })
+  await addDependenciesToPackage({}, ['@pnpm/postinstall-modifies-source@1.0.0'], opts)
+
+  expect(readFileSync('node_modules/@pnpm/postinstall-modifies-source/empty-file.txt', 'utf8')).toContain('hello')
+
+  const cafsDir = path.join(opts.storeDir, 'files')
+  const filesIndexFile = getFilePathInCafs(cafsDir, getIntegrity('@pnpm/postinstall-modifies-source', '1.0.0'), 'index')
+  const filesIndex = await loadJsonFile<PackageFilesIndex>(filesIndexFile)
+  const patchedFileIntegrity = filesIndex.sideEffects?.[`${ENGINE_NAME}-{}`]['empty-file.txt']?.integrity
+  expect(patchedFileIntegrity).toBeTruthy()
+  const originalFileIntegrity = filesIndex.files['empty-file.txt'].integrity
+  expect(originalFileIntegrity).toBeTruthy()
+  // The integrity of the original file differs from the integrity of the patched file
+  expect(originalFileIntegrity).not.toEqual(patchedFileIntegrity)
+
+  expect(readFileSync(getFilePathInCafs(cafsDir, originalFileIntegrity, 'nonexec'), 'utf8')).toEqual('')
+})

packages/create-cafs-store/src/createImportPackage.ts

@@ -19,13 +19,13 @@ interface ImportOptions {
 type ImportFunction = (to: string, opts: ImportOptions) => Promise<string | undefined>
 
 export default (
-  packageImportMethod?: 'auto' | 'hardlink' | 'copy' | 'clone'
+  packageImportMethod?: 'auto' | 'hardlink' | 'copy' | 'clone' | 'clone-or-copy'
 ): ImportFunction => {
   const importPackage = createImportPackage(packageImportMethod)
   return async (to, opts) => limitLinking(async () => importPackage(to, opts))
 }
 
-function createImportPackage (packageImportMethod?: 'auto' | 'hardlink' | 'copy' | 'clone') {
+function createImportPackage (packageImportMethod?: 'auto' | 'hardlink' | 'copy' | 'clone' | 'clone-or-copy') {
   // this works in the following way:
   // - hardlink: hardlink the packages, no fallback
   // - clone: clone the packages, no fallback
@@ -41,6 +41,8 @@ function createImportPackage (packageImportMethod?: 'auto' | 'hardlink' | 'copy'
   case 'auto': {
     return createAutoImporter()
   }
+  case 'clone-or-copy':
+    return createCloneOrCopyImporter()
   case 'copy':
     packageImportMethodLogger.debug({ method: 'copy' })
     return copyPkg
@@ -87,6 +89,29 @@ function createAutoImporter (): ImportFunction {
   }
 }
 
+function createCloneOrCopyImporter (): ImportFunction {
+  let auto = initialAuto
+
+  return async (to, opts) => auto(to, opts)
+
+  async function initialAuto (
+    to: string,
+    opts: ImportOptions
+  ): Promise<string | undefined> {
+    try {
+      if (!await clonePkg(to, opts)) return undefined
+      packageImportMethodLogger.debug({ method: 'clone' })
+      auto = clonePkg
+      return 'clone'
+    } catch (err: any) { // eslint-disable-line
+      // ignore
+    }
+    packageImportMethodLogger.debug({ method: 'copy' })
+    auto = copyPkg
+    return auto(to, opts)
+  }
+}
+
 async function clonePkg (
   to: string,
   opts: ImportOptions

packages/create-cafs-store/src/index.ts

@@ -23,7 +23,10 @@ function createPackageImporter (
   const gfm = getFlatMap.bind(null, opts.cafsDir)
   return async (to, opts) => {
     const { filesMap, isBuilt } = gfm(opts.filesResponse, opts.targetEngine)
-    const impPkg = cachedImporterCreator(opts.filesResponse.packageImportMethod ?? packageImportMethod)
+    const pkgImportMethod = (opts.requiresBuild && !isBuilt)
+      ? 'clone-or-copy'
+      : (opts.filesResponse.packageImportMethod ?? packageImportMethod)
+    const impPkg = cachedImporterCreator(pkgImportMethod)
     const importMethod = await impPkg(to, { filesMap, fromStore: opts.filesResponse.fromStore, force: opts.force })
     return { importMethod, isBuilt }
   }

packages/fetcher-base/src/index.ts

@@ -21,13 +21,16 @@ export type PackageFilesResponse = {
   filesIndex: Record<string, PackageFileInfo>
 })
 
+export interface ImportPackageOpts {
+  requiresBuild?: boolean
+  targetEngine?: string
+  filesResponse: PackageFilesResponse
+  force: boolean
+}
+
 export type ImportPackageFunction = (
   to: string,
-  opts: {
-    targetEngine?: string
-    filesResponse: PackageFilesResponse
-    force: boolean
-  }
+  opts: ImportPackageOpts
 ) => Promise<{ isBuilt: boolean, importMethod: undefined | string }>
 
 export interface Cafs {

packages/headless/package.json

@@ -22,7 +22,7 @@
     "@pnpm/package-store": "workspace:13.0.7",
     "@pnpm/prepare": "workspace:*",
     "@pnpm/read-projects-context": "workspace:6.0.4",
-    "@pnpm/registry-mock": "2.19.0",
+    "@pnpm/registry-mock": "2.20.0",
     "@pnpm/store-path": "workspace:6.0.0",
     "@pnpm/test-fixtures": "workspace:*",
     "@types/fs-extra": "^9.0.5",

packages/headless/src/index.ts

@@ -686,6 +686,7 @@ async function linkAllPkgs (
       const { importMethod, isBuilt } = await storeController.importPackage(depNode.dir, {
         filesResponse,
         force: opts.force,
+        requiresBuild: depNode.requiresBuild,
         targetEngine,
       })
       if (importMethod) {

packages/headless/src/linkHoistedModules.ts

@@ -105,6 +105,7 @@ async function linkAllPkgsInOrder (
       const { importMethod, isBuilt } = await storeController.importPackage(depNode.dir, {
         filesResponse,
         force: opts.force || depNode.depPath !== prevGraph[dir]?.depPath,
+        requiresBuild: depNode.requiresBuild,
         targetEngine,
       })
       if (importMethod) {

packages/package-requester/package.json

@@ -57,6 +57,7 @@
     "promise-share": "^1.0.0",
     "ramda": "^0.27.1",
     "rename-overwrite": "^4.0.2",
+    "safe-promise-defer": "^1.0.1",
     "semver": "^7.3.4",
     "ssri": "^8.0.1"
   },
@@ -65,7 +66,7 @@
     "@pnpm/create-cafs-store": "workspace:1.0.3",
     "@pnpm/logger": "^4.0.0",
     "@pnpm/package-requester": "workspace:18.0.7",
-    "@pnpm/registry-mock": "2.19.0",
+    "@pnpm/registry-mock": "2.20.0",
     "@pnpm/test-fixtures": "workspace:*",
     "@types/normalize-path": "^3.0.0",
     "@types/ramda": "0.27.39",

packages/package-requester/src/packageRequester.ts

@@ -51,7 +51,7 @@ import renameOverwrite from 'rename-overwrite'
 import semver from 'semver'
 import ssri from 'ssri'
 import equalOrSemverEqual from './equalOrSemverEqual'
-import safeDeferredPromise from './safeDeferredPromise'
+import safePromiseDefer from 'safe-promise-defer'
 
 const TARBALL_INTEGRITY_FILENAME = 'tarball-integrity'
 const packageRequestLogger = logger('package-requester')
@@ -441,7 +441,7 @@ function fetchToStore (
 
         if ((pkgFilesIndex?.files) != null) {
           const manifest = opts.fetchRawManifest
-            ? safeDeferredPromise<DependencyManifest | undefined>()
+            ? safePromiseDefer<DependencyManifest | undefined>()
             : undefined
           if (
             (
@@ -498,7 +498,7 @@ Actual package in the store by the given integrity: ${pkgFilesIndex.name}@${pkgF
       const priority = (++ctx.requestsQueue['counter'] % ctx.requestsQueue['concurrency'] === 0 ? -1 : 1) * 1000 // eslint-disable-line
 
       const fetchManifest = opts.fetchRawManifest
-        ? safeDeferredPromise<DependencyManifest | undefined>()
+        ? safePromiseDefer<DependencyManifest | undefined>()
         : undefined
       if (fetchManifest != null) {
         fetchManifest()

packages/plugin-commands-installation/package.json

@@ -39,7 +39,7 @@
     "@pnpm/modules-yaml": "workspace:10.0.2",
     "@pnpm/plugin-commands-installation": "workspace:10.1.1",
     "@pnpm/prepare": "workspace:*",
-    "@pnpm/registry-mock": "2.19.0",
+    "@pnpm/registry-mock": "2.20.0",
     "@pnpm/test-fixtures": "workspace:*",
     "@types/is-ci": "^3.0.0",
     "@types/proxyquire": "^1.3.28",

packages/plugin-commands-listing/package.json

@@ -38,7 +38,7 @@
     "@pnpm/plugin-commands-installation": "workspace:10.1.1",
     "@pnpm/plugin-commands-listing": "workspace:5.0.12",
     "@pnpm/prepare": "workspace:*",
-    "@pnpm/registry-mock": "2.19.0",
+    "@pnpm/registry-mock": "2.20.0",
     "@types/ramda": "0.27.39",
     "execa": "npm:safe-execa@^0.1.1",
     "strip-ansi": "^6.0.0",

packages/plugin-commands-outdated/package.json

@@ -38,7 +38,7 @@
     "@pnpm/plugin-commands-installation": "workspace:10.1.1",
     "@pnpm/plugin-commands-outdated": "workspace:6.0.12",
     "@pnpm/prepare": "workspace:*",
-    "@pnpm/registry-mock": "2.19.0",
+    "@pnpm/registry-mock": "2.20.0",
     "@types/lru-cache": "^5.1.0",
     "@types/ramda": "0.27.39",
     "@types/wrap-ansi": "^3.0.0",

packages/plugin-commands-publishing/package.json

@@ -39,7 +39,7 @@
     "@pnpm/logger": "^4.0.0",
     "@pnpm/plugin-commands-publishing": "workspace:5.0.13",
     "@pnpm/prepare": "workspace:*",
-    "@pnpm/registry-mock": "2.19.0",
+    "@pnpm/registry-mock": "2.20.0",
     "@types/cross-spawn": "^6.0.2",
     "@types/is-ci": "^3.0.0",
     "@types/is-windows": "^1.0.0",

packages/plugin-commands-rebuild/package.json

@@ -37,7 +37,7 @@
     "@pnpm/logger": "^4.0.0",
     "@pnpm/plugin-commands-rebuild": "workspace:6.1.11",
     "@pnpm/prepare": "workspace:*",
-    "@pnpm/registry-mock": "2.19.0",
+    "@pnpm/registry-mock": "2.20.0",
     "@pnpm/test-fixtures": "workspace:*",
     "@types/ramda": "0.27.39",
     "@types/semver": "^7.3.4",

packages/plugin-commands-script-runners/package.json

@@ -38,7 +38,7 @@
     "@pnpm/logger": "^4.0.0",
     "@pnpm/plugin-commands-script-runners": "workspace:5.0.15",
     "@pnpm/prepare": "workspace:*",
-    "@pnpm/registry-mock": "2.19.0",
+    "@pnpm/registry-mock": "2.20.0",
     "@types/is-windows": "^1.0.0",
     "@types/ramda": "0.27.39",
     "is-windows": "^1.0.2",

packages/plugin-commands-store/package.json

@@ -38,7 +38,7 @@
     "@pnpm/logger": "^4.0.0",
     "@pnpm/plugin-commands-store": "workspace:5.1.11",
     "@pnpm/prepare": "workspace:*",
-    "@pnpm/registry-mock": "2.19.0",
+    "@pnpm/registry-mock": "2.20.0",
     "@types/archy": "0.0.31",
     "@types/ramda": "0.27.39",
     "@types/ssri": "^7.1.0",

packages/pnpm/package.json

@@ -55,7 +55,7 @@
     "@pnpm/prepare": "workspace:*",
     "@pnpm/read-package-json": "workspace:6.0.3",
     "@pnpm/read-project-manifest": "workspace:3.0.3",
-    "@pnpm/registry-mock": "2.19.0",
+    "@pnpm/registry-mock": "2.20.0",
     "@pnpm/run-npm": "workspace:4.0.1",
     "@pnpm/tabtab": "^0.1.2",
     "@pnpm/types": "workspace:8.1.0",

packages/resolve-dependencies/package.json

@@ -51,9 +51,11 @@
     "is-inner-link": "^4.0.0",
     "is-subdir": "^1.1.1",
     "path-exists": "^4.0.0",
+    "promise-share": "^1.0.0",
     "ramda": "^0.27.1",
     "rename-overwrite": "^4.0.2",
     "replace-string": "^3.1.0",
+    "safe-promise-defer": "^1.0.1",
     "semver": "^7.3.4",
     "semver-range-intersect": "^0.3.1",
     "version-selector-type": "^3.0.0"