feat: include some settings as fields in the lockfile (#6557)

ref #6312
close #6609

.changeset/fast-news-deliver.md

@@ -0,0 +1,15 @@
+---
+"@pnpm/lockfile-types": minor
+"@pnpm/lockfile-file": minor
+"@pnpm/core": minor
+"pnpm": minor
+---
+
+Some settings influence the structure of the lockfile, so we cannot reuse the lockfile if those settings change. As a result, we need to store such settings in the lockfile. This way we will know with which settings the lockfile has been created.
+
+A new field will now be present in the lockfile: `settings`. It will store the values of two settings: `autoInstallPeers` and `excludeLinksFromLockfile`. If someone tries to perform a `frozen-lockfile` installation and their active settings don't match the ones in the lockfile, then an error message will be thrown.
+
+The lockfile format version is bumped from v6.0 to v6.1.
+
+Related PR: [#6557](https://github.com/pnpm/pnpm/pull/6557)
+Related issue: [#6312](https://github.com/pnpm/pnpm/issues/6312)

.changeset/itchy-moons-approve.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/lockfile-file": patch
+---
+
+Convertion should work for all lockfile v6 formats, not just 6.0.

.changeset/sharp-trees-pretend.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/constants": minor
+---
+
+Bump lockfile v6 version to v6.1.

.changeset/short-coats-fetch.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/plugin-commands-patching": patch
+---
+
+Don't run install with the `frozen-lockfile=true` setting.

.changeset/tender-candles-sleep.md

@@ -0,0 +1,7 @@
+---
+"@pnpm/plugin-commands-rebuild": major
+"@pnpm/plugin-commands-store": major
+"@pnpm/get-context": major
+---
+
+New required options added: autoInstallPeers and excludeLinksFromLockfile.

exec/plugin-commands-rebuild/src/implementation/extendRebuildOptions.ts

@@ -6,8 +6,10 @@ import { type Registries } from '@pnpm/types'
 import loadJsonFile from 'load-json-file'
 
 export interface StrictRebuildOptions {
+  autoInstallPeers: boolean
   cacheDir: string
   childConcurrency: number
+  excludeLinksFromLockfile: boolean
   extraBinPaths: string[]
   extraEnv: Record<string, string>
   lockfileDir: string

lockfile/lockfile-file/src/experiments/inlineSpecifiersLockfileConverters.ts

@@ -113,7 +113,7 @@ export function revertFromInlineSpecifiersFormat (lockfile: InlineSpecifiersLock
 
   let revertedImporters = mapValues(importers, revertProjectSnapshot)
   let packages = lockfile.packages
-  if (originalVersion === 6) {
+  if (originalVersionStr.startsWith('6.')) {
     revertedImporters = Object.fromEntries(
       Object.entries(revertedImporters ?? {})
         .map(([importerId, pkgSnapshot]: [string, ProjectSnapshot]) => {
@@ -150,7 +150,7 @@ export function revertFromInlineSpecifiersFormat (lockfile: InlineSpecifiersLock
     packages,
     importers: revertedImporters,
   }
-  if (originalVersion === 6 && newLockfile.time) {
+  if (originalVersionStr.startsWith('6.') && newLockfile.time) {
     newLockfile.time = Object.fromEntries(
       Object.entries(newLockfile.time)
         .map(([depPath, time]) => [convertLockfileV6DepPathToV5DepPath(depPath), time])

lockfile/lockfile-file/src/read.ts

@@ -136,6 +136,7 @@ export function createLockfileObject (
   importerIds: string[],
   opts: {
     lockfileVersion: number | string
+    autoInstallPeers: boolean
   }
 ) {
   const importers = importerIds.reduce((acc, importerId) => {
@@ -148,6 +149,10 @@ export function createLockfileObject (
   return {
     importers,
     lockfileVersion: opts.lockfileVersion || LOCKFILE_VERSION,
+    settings: {
+      autoInstallPeers: opts.autoInstallPeers,
+      excludeLinksFromLockfile: false,
+    },
   }
 }
 

lockfile/lockfile-file/src/sortLockfileKeys.ts

@@ -33,12 +33,13 @@ const ORDERED_KEYS = {
 
 const ROOT_KEYS_ORDER = {
   lockfileVersion: 1,
+  settings: 2,
   // only and never are conflict options.
-  neverBuiltDependencies: 2,
-  onlyBuiltDependencies: 2,
-  overrides: 3,
-  packageExtensionsChecksum: 4,
-  patchedDependencies: 5,
+  neverBuiltDependencies: 3,
+  onlyBuiltDependencies: 3,
+  overrides: 4,
+  packageExtensionsChecksum: 5,
+  patchedDependencies: 6,
   specifiers: 10,
   dependencies: 11,
   optionalDependencies: 12,

lockfile/lockfile-types/src/index.ts

@@ -2,6 +2,11 @@ import { type DependenciesMeta, type PatchFile } from '@pnpm/types'
 
 export type { PatchFile }
 
+export interface LockfileSettings {
+  autoInstallPeers?: boolean
+  excludeLinksFromLockfile?: boolean
+}
+
 export interface Lockfile {
   importers: Record<string, ProjectSnapshot>
   lockfileVersion: number | string
@@ -12,6 +17,7 @@ export interface Lockfile {
   overrides?: Record<string, string>
   packageExtensionsChecksum?: string
   patchedDependencies?: Record<string, PatchFile>
+  settings?: LockfileSettings
 }
 
 export interface ProjectSnapshot {

packages/constants/src/index.ts

@@ -1,6 +1,6 @@
 export const WANTED_LOCKFILE = 'pnpm-lock.yaml'
 export const LOCKFILE_VERSION = 5.4
-export const LOCKFILE_VERSION_V6 = '6.0'
+export const LOCKFILE_VERSION_V6 = '6.1'
 
 export const ENGINE_NAME = `${process.platform}-${process.arch}-node-${process.version.split('.')[0]}`
 export const LAYOUT_VERSION = 5

patching/plugin-commands-patching/src/patchCommit.ts

@@ -74,7 +74,13 @@ export async function handler (opts: install.InstallCommandOptions & Pick<Config
     opts.allProjectsGraph[lockfileDir].package.manifest = rootProjectManifest
   }
 
-  return install.handler(opts)
+  return install.handler({
+    ...opts,
+    rawLocalConfig: {
+      ...opts.rawLocalConfig,
+      'frozen-lockfile': false,
+    },
+  })
 }
 
 async function diffFolders (folderA: string, folderB: string) {

patching/plugin-commands-patching/test/patch.test.ts

@@ -516,18 +516,24 @@ describe('patch and commit in workspaces', () => {
 })
 
 describe('patch with custom modules-dir and virtual-store-dir', () => {
-  const customModulesDirFixture = tempDir()
-  f.copy('custom-modules-dir', customModulesDirFixture)
-  const cacheDir = path.resolve(customModulesDirFixture, 'cache')
-  const storeDir = path.resolve(customModulesDirFixture, 'store')
-  const defaultPatchOption = {
-    ...basePatchOption,
-    cacheDir,
-    dir: customModulesDirFixture,
-    storeDir,
-    modulesDir: 'fake_modules',
-    virtualStoreDir: 'fake_modules/.fake_store',
-  }
+  let defaultPatchOption: patch.PatchCommandOptions
+  let customModulesDirFixture: string
+  let cacheDir: string
+  let storeDir: string
+  beforeAll(() => {
+    customModulesDirFixture = tempDir()
+    f.copy('custom-modules-dir', customModulesDirFixture)
+    cacheDir = path.resolve(customModulesDirFixture, 'cache')
+    storeDir = path.resolve(customModulesDirFixture, 'store')
+    defaultPatchOption = {
+      ...basePatchOption,
+      cacheDir,
+      dir: customModulesDirFixture,
+      storeDir,
+      modulesDir: 'fake_modules',
+      virtualStoreDir: 'fake_modules/.fake_store',
+    }
+  })
 
   test('should work with custom modules-dir and virtual-store-dir', async () => {
     const manifest = fs.readFileSync(path.join(customModulesDirFixture, 'package.json'), 'utf8')

pkg-manager/core/src/getPeerDependencyIssues.ts

@@ -20,7 +20,7 @@ export type ListMissingPeersOptions = Partial<GetContextOptions>
 | 'useGitBranchLockfile'
 | 'workspacePackages'
 >
-& Pick<GetContextOptions, 'storeDir'>
+& Pick<GetContextOptions, 'autoInstallPeers' | 'storeDir'>
 
 export async function getPeerDependencyIssues (
   projects: ProjectOptions[],

pkg-manager/core/src/install/index.ts

@@ -83,6 +83,15 @@ import { getAllUniqueSpecs, getPreferredVersionsFromLockfileAndManifests } from
 import { linkPackages } from './link'
 import { reportPeerDependencyIssues } from './reportPeerDependencyIssues'
 
+class LockfileConfigMismatchError extends PnpmError {
+  constructor (outdatedLockfileSettingName: string) {
+    super('LOCKFILE_CONFIG_MISMATCH',
+      `Cannot proceed with the frozen installation. The current "${outdatedLockfileSettingName!}" configuration doesn't match the value found in the lockfile`, {
+        hint: 'Update your lockfile using "pnpm install --no-frozen-lockfile"',
+      })
+  }
+}
+
 const BROKEN_LOCKFILE_INTEGRITY_ERRORS = new Set([
   'ERR_PNPM_UNEXPECTED_PKG_CONTENT_IN_STORE',
   'ERR_PNPM_TARBALL_INTEGRITY',
@@ -311,25 +320,42 @@ export async function mutateModules (
     if (opts.useLockfileV6 == null) {
       opts.useLockfileV6 = ctx.wantedLockfile.lockfileVersion.toString().startsWith('6.')
     }
-    let needsFullResolution = !maybeOpts.ignorePackageManifest &&
-      lockfileIsNotUpToDate(ctx.wantedLockfile, {
+    const frozenLockfile = opts.frozenLockfile ||
+      opts.frozenLockfileIfExists && ctx.existsWantedLockfile
+    let outdatedLockfileSettings = false
+    if (!opts.ignorePackageManifest) {
+      const outdatedLockfileSettingName = getOutdatedLockfileSetting(ctx.wantedLockfile, {
+        autoInstallPeers: opts.autoInstallPeers,
         overrides: opts.overrides,
         neverBuiltDependencies: opts.neverBuiltDependencies,
         onlyBuiltDependencies: opts.onlyBuiltDependencies,
         packageExtensionsChecksum,
         patchedDependencies,
-      }) ||
+      })
+      outdatedLockfileSettings = outdatedLockfileSettingName != null
+      if (frozenLockfile && outdatedLockfileSettings) {
+        throw new LockfileConfigMismatchError(outdatedLockfileSettingName!)
+      }
+    }
+    let needsFullResolution = outdatedLockfileSettings ||
       opts.fixLockfile ||
       opts.useLockfileV6 && !ctx.wantedLockfile.lockfileVersion.toString().startsWith('6.')
     if (needsFullResolution) {
+      ctx.wantedLockfile.settings = {
+        autoInstallPeers: opts.autoInstallPeers,
+        excludeLinksFromLockfile: false,
+      }
       ctx.wantedLockfile.overrides = opts.overrides
       ctx.wantedLockfile.neverBuiltDependencies = opts.neverBuiltDependencies
       ctx.wantedLockfile.onlyBuiltDependencies = opts.onlyBuiltDependencies
       ctx.wantedLockfile.packageExtensionsChecksum = packageExtensionsChecksum
       ctx.wantedLockfile.patchedDependencies = patchedDependencies
+    } else if (!frozenLockfile) {
+      ctx.wantedLockfile.settings = {
+        autoInstallPeers: opts.autoInstallPeers,
+        excludeLinksFromLockfile: false,
+      }
     }
-    const frozenLockfile = opts.frozenLockfile ||
-      opts.frozenLockfileIfExists && ctx.existsWantedLockfile
     if (
       !ctx.lockfileHadConflicts &&
       !opts.fixLockfile &&
@@ -619,26 +645,46 @@ async function calcPatchHashes (patches: Record<string, string>, lockfileDir: st
   }, patches)
 }
 
-function lockfileIsNotUpToDate (
+function getOutdatedLockfileSetting (
   lockfile: Lockfile,
   {
     neverBuiltDependencies,
     onlyBuiltDependencies,
     overrides,
     packageExtensionsChecksum,
     patchedDependencies,
+    autoInstallPeers,
   }: {
     neverBuiltDependencies?: string[]
     onlyBuiltDependencies?: string[]
     overrides?: Record<string, string>
     packageExtensionsChecksum?: string
     patchedDependencies?: Record<string, PatchFile>
-  }) {
-  return !equals(lockfile.overrides ?? {}, overrides ?? {}) ||
-    !equals((lockfile.neverBuiltDependencies ?? []).sort(), (neverBuiltDependencies ?? []).sort()) ||
-    !equals(onlyBuiltDependencies?.sort(), lockfile.onlyBuiltDependencies) ||
-    lockfile.packageExtensionsChecksum !== packageExtensionsChecksum ||
-    !equals(lockfile.patchedDependencies ?? {}, patchedDependencies ?? {})
+    autoInstallPeers?: boolean
+  }
+) {
+  if (!equals(lockfile.overrides ?? {}, overrides ?? {})) {
+    return 'overrides'
+  }
+  if (!equals((lockfile.neverBuiltDependencies ?? []).sort(), (neverBuiltDependencies ?? []).sort())) {
+    return 'neverBuiltDependencies'
+  }
+  if (!equals(onlyBuiltDependencies?.sort(), lockfile.onlyBuiltDependencies)) {
+    return 'onlyBuiltDependencies'
+  }
+  if (lockfile.packageExtensionsChecksum !== packageExtensionsChecksum) {
+    return 'packageExtensionsChecksum'
+  }
+  if (!equals(lockfile.patchedDependencies ?? {}, patchedDependencies ?? {})) {
+    return 'patchedDependencies'
+  }
+  if ((lockfile.settings?.autoInstallPeers != null && lockfile.settings.autoInstallPeers !== autoInstallPeers)) {
+    return 'settings.autoInstallPeers'
+  }
+  if (lockfile.settings?.excludeLinksFromLockfile) {
+    return 'settings.excludeLinksFromLockfile'
+  }
+  return null
 }
 
 export function createObjectChecksum (obj: unknown) {

pkg-manager/core/src/link/options.ts

@@ -9,7 +9,9 @@ import {
 import { type ReporterFunction } from '../types'
 
 interface StrictLinkOptions {
+  autoInstallPeers: boolean
   binsDir: string
+  excludeLinksFromLockfile: boolean
   force: boolean
   forceSharedLockfile: boolean
   useLockfile: boolean

pkg-manager/core/test/install/aliases.ts

@@ -13,6 +13,10 @@ test('installing aliased dependency', async () => {
   expect(typeof project.requireModule('positive')).toBe('function')
 
   expect(await project.readLockfile()).toStrictEqual({
+    settings: {
+      autoInstallPeers: true,
+      excludeLinksFromLockfile: false,
+    },
     dependencies: {
       negative: '/is-negative/1.0.0',
       positive: '/is-positive/3.1.0',
@@ -68,6 +72,10 @@ test('a dependency has an aliased subdependency', async () => {
   expect(project.requireModule('@pnpm.e2e/pkg-with-1-aliased-dep')().name).toEqual('@pnpm.e2e/dep-of-pkg-with-1-dep')
 
   expect(await project.readLockfile()).toStrictEqual({
+    settings: {
+      autoInstallPeers: true,
+      excludeLinksFromLockfile: false,
+    },
     dependencies: {
       '@pnpm.e2e/pkg-with-1-aliased-dep': '100.0.0',
     },

pkg-manager/core/test/install/frozenLockfile.ts

@@ -287,3 +287,18 @@ test('prefer-frozen-lockfile: should prefer frozen-lockfile when package has lin
   await projects['p1'].has('p2')
   await projects['p2'].has('is-negative')
 })
+
+test('frozen-lockfile: installation fails if the value of auto-install-peers changes', async () => {
+  prepareEmpty()
+  const manifest = {
+    dependencies: {
+      'is-positive': '^3.0.0',
+    },
+  }
+
+  await install(manifest, await testDefaults({ autoInstallPeers: true }))
+
+  await expect(
+    install(manifest, await testDefaults({ frozenLockfile: true, autoInstallPeers: false }))
+  ).rejects.toThrow('Cannot proceed with the frozen installation. The current "settings.autoInstallPeers" configuration doesn\'t match the value found in the lockfile')
+})

pkg-manager/core/test/install/local.ts

@@ -45,6 +45,10 @@ test('local file', async () => {
   const lockfile = await project.readLockfile()
 
   expect(lockfile).toStrictEqual({
+    settings: {
+      autoInstallPeers: true,
+      excludeLinksFromLockfile: false,
+    },
     dependencies: {
       'local-pkg': 'link:../local-pkg',
     },
@@ -86,6 +90,10 @@ test('local file via link:', async () => {
   const lockfile = await project.readLockfile()
 
   expect(lockfile).toStrictEqual({
+    settings: {
+      autoInstallPeers: true,
+      excludeLinksFromLockfile: false,
+    },
     dependencies: {
       'local-pkg': 'link:../local-pkg',
     },
@@ -112,6 +120,10 @@ test('local file with symlinked node_modules', async () => {
   const lockfile = await project.readLockfile()
 
   expect(lockfile).toStrictEqual({
+    settings: {
+      autoInstallPeers: true,
+      excludeLinksFromLockfile: false,
+    },
     dependencies: {
       'local-pkg': 'link:../local-pkg',
     },