feat!: lockfile format v6

.changeset/orange-cobras-joke.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/constants": minor
+---
+
+Exported a constant for the new lockfile format version: `LOCKFILE_FORMAT_V6`.

.changeset/orange-pears-flow.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/lockfile-file": major
+---
+
+Breaking change to the API of the read functions. Instead of one wanted lockfile version, it now expects an array of `wantedVersions`.

.changeset/shy-balloons-build.md

@@ -0,0 +1,10 @@
+---
+"@pnpm/plugin-commands-installation": minor
+"@pnpm/resolve-dependencies": minor
+"@pnpm/get-context": minor
+"@pnpm/core": minor
+"@pnpm/config": minor
+"pnpm": minor
+---
+
+Added support for `pnpm-lock.yaml` format v6. This new format will be the new lockfile format in pnpm v8. To use the new lockfile format, use the `use-lockfile-v6=true` setting in `.npmrc`. Or run `pnpm install --use-lockfile-v6` [#5810](https://github.com/pnpm/pnpm/pull/5810).

.changeset/thick-dingos-enjoy.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/dependency-path": minor
+---
+
+Updated the functions to support dependency paths used in the 6th version of the lockfile. Exported a new function: createPeersFolderSuffixNewFormat.

config/config/src/Config.ts

@@ -88,6 +88,7 @@ export interface Config {
   registrySupportsTimeField?: boolean
   failedToLoadBuiltInConfig: boolean
   resolvePeersFromWorkspaceRoot?: boolean
+  useLockfileV6?: boolean
 
   // proxy
   httpProxy?: string

config/config/src/index.ts

@@ -111,6 +111,7 @@ export const types = Object.assign({
   'strict-peer-dependencies': Boolean,
   'use-beta-cli': Boolean,
   'use-inline-specifiers-lockfile-format': Boolean,
+  'use-lockfile-v6': Boolean,
   'use-node-version': String,
   'use-running-store-server': Boolean,
   'use-store-server': Boolean,

lockfile/lockfile-file/src/experiments/inlineSpecifiersLockfileConverters.ts

@@ -1,3 +1,4 @@
+import * as dp from '@pnpm/dependency-path'
 import type { Lockfile, ProjectSnapshot, ResolvedDependencies } from '@pnpm/lockfile-types'
 import {
   INLINE_SPECIFIERS_FORMAT_LOCKFILE_VERSION_SUFFIX,
@@ -10,15 +11,89 @@ export function isExperimentalInlineSpecifiersFormat (
   lockfile: InlineSpecifiersLockfile | Lockfile
 ): lockfile is InlineSpecifiersLockfile {
   const { lockfileVersion } = lockfile
-  return typeof lockfileVersion === 'string' && lockfileVersion.endsWith(INLINE_SPECIFIERS_FORMAT_LOCKFILE_VERSION_SUFFIX)
+  return lockfileVersion.toString().startsWith('6.') || typeof lockfileVersion === 'string' && lockfileVersion.endsWith(INLINE_SPECIFIERS_FORMAT_LOCKFILE_VERSION_SUFFIX)
 }
 
 export function convertToInlineSpecifiersFormat (lockfile: Lockfile): InlineSpecifiersLockfile {
-  return {
+  let importers = lockfile.importers
+  let packages = lockfile.packages
+  if (lockfile.lockfileVersion.toString().startsWith('6.')) {
+    importers = Object.fromEntries(
+      Object.entries(lockfile.importers ?? {})
+        .map(([importerId, pkgSnapshot]: [string, ProjectSnapshot]) => {
+          const newSnapshot = { ...pkgSnapshot }
+          if (newSnapshot.dependencies != null) {
+            newSnapshot.dependencies = mapValues(newSnapshot.dependencies, convertOldRefToNewRef)
+          }
+          if (newSnapshot.optionalDependencies != null) {
+            newSnapshot.optionalDependencies = mapValues(newSnapshot.optionalDependencies, convertOldRefToNewRef)
+          }
+          if (newSnapshot.devDependencies != null) {
+            newSnapshot.devDependencies = mapValues(newSnapshot.devDependencies, convertOldRefToNewRef)
+          }
+          return [importerId, newSnapshot]
+        })
+    )
+    packages = Object.fromEntries(
+      Object.entries(lockfile.packages ?? {})
+        .map(([depPath, pkgSnapshot]) => {
+          const newSnapshot = { ...pkgSnapshot }
+          if (newSnapshot.dependencies != null) {
+            newSnapshot.dependencies = mapValues(newSnapshot.dependencies, convertOldRefToNewRef)
+          }
+          if (newSnapshot.optionalDependencies != null) {
+            newSnapshot.optionalDependencies = mapValues(newSnapshot.optionalDependencies, convertOldRefToNewRef)
+          }
+          return [convertOldDepPathToNewDepPath(depPath), newSnapshot]
+        })
+    )
+  }
+  const newLockfile = {
     ...lockfile,
-    lockfileVersion: `${lockfile.lockfileVersion}${INLINE_SPECIFIERS_FORMAT_LOCKFILE_VERSION_SUFFIX}`,
-    importers: mapValues(lockfile.importers, convertProjectSnapshotToInlineSpecifiersFormat),
+    packages,
+    lockfileVersion: lockfile.lockfileVersion.toString().startsWith('6.')
+      ? lockfile.lockfileVersion.toString()
+      : (
+        lockfile.lockfileVersion.toString().endsWith(INLINE_SPECIFIERS_FORMAT_LOCKFILE_VERSION_SUFFIX)
+          ? lockfile.lockfileVersion.toString()
+          : `${lockfile.lockfileVersion}${INLINE_SPECIFIERS_FORMAT_LOCKFILE_VERSION_SUFFIX}`
+      ),
+    importers: mapValues(importers, convertProjectSnapshotToInlineSpecifiersFormat),
+  }
+  if (lockfile.lockfileVersion.toString().startsWith('6.') && newLockfile.time) {
+    newLockfile.time = Object.fromEntries(
+      Object.entries(newLockfile.time)
+        .map(([depPath, time]) => [convertOldDepPathToNewDepPath(depPath), time])
+    )
   }
+  return newLockfile
+}
+
+function convertOldDepPathToNewDepPath (oldDepPath: string) {
+  const parsedDepPath = dp.parse(oldDepPath)
+  if (!parsedDepPath.name || !parsedDepPath.version) return oldDepPath
+  let newDepPath = `/${parsedDepPath.name}@${parsedDepPath.version}`
+  if (parsedDepPath.peersSuffix) {
+    if (parsedDepPath.peersSuffix.startsWith('(')) {
+      newDepPath += parsedDepPath.peersSuffix
+    } else {
+      newDepPath += `_${parsedDepPath.peersSuffix}`
+    }
+  }
+  if (parsedDepPath.host) {
+    newDepPath = `${parsedDepPath.host}${newDepPath}`
+  }
+  return newDepPath
+}
+
+function convertOldRefToNewRef (oldRef: string) {
+  if (oldRef.startsWith('link:') || oldRef.startsWith('file:')) {
+    return oldRef
+  }
+  if (oldRef.includes('/')) {
+    return convertOldDepPathToNewDepPath(oldRef)
+  }
+  return oldRef
 }
 
 export function revertFromInlineSpecifiersFormatIfNecessary (lockfile: Lockfile | InlineSpecifiersLockfile): Lockfile {
@@ -36,11 +111,69 @@ export function revertFromInlineSpecifiersFormat (lockfile: InlineSpecifiersLock
     throw new Error(`Unable to revert lockfile from inline specifiers format. Invalid version parsed: ${originalVersionStr}`)
   }
 
-  return {
+  let revertedImporters = mapValues(importers, revertProjectSnapshot)
+  let packages = lockfile.packages
+  if (originalVersion === 6) {
+    revertedImporters = Object.fromEntries(
+      Object.entries(revertedImporters ?? {})
+        .map(([importerId, pkgSnapshot]: [string, ProjectSnapshot]) => {
+          const newSnapshot = { ...pkgSnapshot }
+          if (newSnapshot.dependencies != null) {
+            newSnapshot.dependencies = mapValues(newSnapshot.dependencies, convertNewRefToOldRef)
+          }
+          if (newSnapshot.optionalDependencies != null) {
+            newSnapshot.optionalDependencies = mapValues(newSnapshot.optionalDependencies, convertNewRefToOldRef)
+          }
+          if (newSnapshot.devDependencies != null) {
+            newSnapshot.devDependencies = mapValues(newSnapshot.devDependencies, convertNewRefToOldRef)
+          }
+          return [importerId, newSnapshot]
+        })
+    )
+    packages = Object.fromEntries(
+      Object.entries(lockfile.packages ?? {})
+        .map(([depPath, pkgSnapshot]) => {
+          const newSnapshot = { ...pkgSnapshot }
+          if (newSnapshot.dependencies != null) {
+            newSnapshot.dependencies = mapValues(newSnapshot.dependencies, convertNewRefToOldRef)
+          }
+          if (newSnapshot.optionalDependencies != null) {
+            newSnapshot.optionalDependencies = mapValues(newSnapshot.optionalDependencies, convertNewRefToOldRef)
+          }
+          return [convertNewDepPathToOldDepPath(depPath), newSnapshot]
+        })
+    )
+  }
+  const newLockfile = {
     ...rest,
-    lockfileVersion: originalVersion,
-    importers: mapValues(importers, revertProjectSnapshot),
+    lockfileVersion: lockfileVersion.endsWith(INLINE_SPECIFIERS_FORMAT_LOCKFILE_VERSION_SUFFIX) ? originalVersion : lockfileVersion,
+    packages,
+    importers: revertedImporters,
+  }
+  if (originalVersion === 6 && newLockfile.time) {
+    newLockfile.time = Object.fromEntries(
+      Object.entries(newLockfile.time)
+        .map(([depPath, time]) => [convertNewDepPathToOldDepPath(depPath), time])
+    )
+  }
+  return newLockfile
+}
+
+export function convertNewDepPathToOldDepPath (oldDepPath: string) {
+  if (!oldDepPath.includes('@', 2)) return oldDepPath
+  const index = oldDepPath.indexOf('@', oldDepPath.indexOf('/@') + 2)
+  if (oldDepPath.includes('(') && index > oldDepPath.indexOf('(')) return oldDepPath
+  return `${oldDepPath.substring(0, index)}/${oldDepPath.substring(index + 1)}`
+}
+
+function convertNewRefToOldRef (oldRef: string) {
+  if (oldRef.startsWith('link:') || oldRef.startsWith('file:')) {
+    return oldRef
+  }
+  if (oldRef.includes('@')) {
+    return convertNewDepPathToOldDepPath(oldRef)
   }
+  return oldRef
 }
 
 function convertProjectSnapshotToInlineSpecifiersFormat (

lockfile/lockfile-file/src/read.ts

@@ -23,7 +23,7 @@ import { revertFromInlineSpecifiersFormatIfNecessary } from './experiments/inlin
 export async function readCurrentLockfile (
   virtualStoreDir: string,
   opts: {
-    wantedVersion?: number
+    wantedVersions?: string[]
     ignoreIncompatible: boolean
   }
 ): Promise<Lockfile | null> {
@@ -34,7 +34,7 @@ export async function readCurrentLockfile (
 export async function readWantedLockfileAndAutofixConflicts (
   pkgPath: string,
   opts: {
-    wantedVersion?: number
+    wantedVersions?: string[]
     ignoreIncompatible: boolean
     useGitBranchLockfile?: boolean
     mergeGitBranchLockfiles?: boolean
@@ -52,7 +52,7 @@ export async function readWantedLockfileAndAutofixConflicts (
 export async function readWantedLockfile (
   pkgPath: string,
   opts: {
-    wantedVersion?: number
+    wantedVersions?: string[]
     ignoreIncompatible: boolean
     useGitBranchLockfile?: boolean
     mergeGitBranchLockfiles?: boolean
@@ -66,7 +66,7 @@ async function _read (
   prefix: string, // only for logging
   opts: {
     autofixMergeConflicts?: boolean
-    wantedVersion?: number
+    wantedVersions?: string[]
     ignoreIncompatible: boolean
   }
 ): Promise<{
@@ -105,14 +105,21 @@ async function _read (
     const lockfile = revertFromInlineSpecifiersFormatIfNecessary(convertFromLockfileFileMutable(lockfileFile))
     const lockfileSemver = comverToSemver((lockfile.lockfileVersion ?? 0).toString())
     /* eslint-enable @typescript-eslint/dot-notation */
-    if (typeof opts.wantedVersion !== 'number' || semver.major(lockfileSemver) === semver.major(comverToSemver(opts.wantedVersion.toString()))) {
-      if (typeof opts.wantedVersion === 'number' && semver.gt(lockfileSemver, comverToSemver(opts.wantedVersion.toString()))) {
-        logger.warn({
-          message: `Your ${WANTED_LOCKFILE} was generated by a newer version of pnpm. ` +
-            `It is a compatible version but it might get downgraded to version ${opts.wantedVersion}`,
-          prefix,
-        })
-      }
+    if (
+      !opts.wantedVersions ||
+      opts.wantedVersions.length === 0 ||
+      opts.wantedVersions.some((wantedVersion) => {
+        if (semver.major(lockfileSemver) !== semver.major(comverToSemver(wantedVersion))) return false
+        if (semver.gt(lockfileSemver, comverToSemver(wantedVersion))) {
+          logger.warn({
+            message: `Your ${WANTED_LOCKFILE} was generated by a newer version of pnpm. ` +
+              `It is a compatible version but it might get downgraded to version ${wantedVersion}`,
+            prefix,
+          })
+        }
+        return true
+      })
+    ) {
       return { lockfile, hadConflicts }
     }
   }
@@ -129,7 +136,7 @@ async function _read (
 export function createLockfileObject (
   importerIds: string[],
   opts: {
-    lockfileVersion: number
+    lockfileVersion: number | string
   }
 ) {
   const importers = importerIds.reduce((acc, importerId) => {
@@ -148,7 +155,7 @@ export function createLockfileObject (
 async function _readWantedLockfile (
   pkgPath: string,
   opts: {
-    wantedVersion?: number
+    wantedVersions?: string[]
     ignoreIncompatible: boolean
     useGitBranchLockfile?: boolean
     mergeGitBranchLockfiles?: boolean
@@ -184,7 +191,7 @@ async function _mergeGitBranchLockfiles (
   prefix: string,
   opts: {
     autofixMergeConflicts?: boolean
-    wantedVersion?: number
+    wantedVersions?: string[]
     ignoreIncompatible: boolean
   }
 ): Promise<Lockfile | null> {
@@ -210,7 +217,7 @@ async function _readGitBranchLockfiles (
   prefix: string,
   opts: {
     autofixMergeConflicts?: boolean
-    wantedVersion?: number
+    wantedVersions?: string[]
     ignoreIncompatible: boolean
   }
 ): Promise<Array<{

lockfile/lockfile-file/src/write.ts

@@ -71,7 +71,7 @@ async function writeLockfile (
     return rimraf(lockfilePath)
   }
 
-  const lockfileToStringify = (opts?.useInlineSpecifiersFormat ?? false)
+  const lockfileToStringify = (Boolean(opts?.useInlineSpecifiersFormat) || wantedLockfile['lockfileVersion'].toString().startsWith('6.'))
     ? convertToInlineSpecifiersFormat(wantedLockfile) as unknown as Lockfile
     : wantedLockfile
 
@@ -143,7 +143,7 @@ export function normalizeLockfile (lockfile: Lockfile, opts: NormalizeLockfileOp
     }
   }
   if (lockfileToSave.time) {
-    lockfileToSave.time = pruneTime(lockfileToSave.time, lockfile.importers)
+    lockfileToSave.time = (lockfileToSave.lockfileVersion.toString().startsWith('6.') ? pruneTimeInLockfileV6 : pruneTime)(lockfileToSave.time, lockfile.importers)
   }
   if ((lockfileToSave.overrides != null) && isEmpty(lockfileToSave.overrides)) {
     delete lockfileToSave.overrides
@@ -167,6 +167,41 @@ export function normalizeLockfile (lockfile: Lockfile, opts: NormalizeLockfileOp
   return lockfileToSave
 }
 
+function pruneTimeInLockfileV6 (time: Record<string, string>, importers: Record<string, ProjectSnapshot>): Record<string, string> {
+  const rootDepPaths = new Set<string>()
+  for (const importer of Object.values(importers)) {
+    for (const depType of DEPENDENCIES_FIELDS) {
+      for (let [depName, ref] of Object.entries(importer[depType] ?? {})) {
+        if (ref['version']) {
+          ref = ref['version']
+        }
+        const suffixStart = ref.indexOf('(')
+        const refWithoutPeerSuffix = suffixStart === -1 ? ref : ref.slice(0, suffixStart)
+        const depPath = refToRelative(refWithoutPeerSuffix, depName)
+        if (!depPath) continue
+        rootDepPaths.add(depPath)
+      }
+    }
+  }
+  return pickBy((_, depPath) => rootDepPaths.has(depPath), time)
+}
+
+function refToRelative (
+  reference: string,
+  pkgName: string
+): string | null {
+  if (reference.startsWith('link:')) {
+    return null
+  }
+  if (reference.startsWith('file:')) {
+    return reference
+  }
+  if (!reference.includes('/') || !reference.replace(/(\([^)]+\))+$/, '').includes('/')) {
+    return `/${pkgName}@${reference}`
+  }
+  return reference
+}
+
 function pruneTime (time: Record<string, string>, importers: Record<string, ProjectSnapshot>): Record<string, string> {
   const rootDepPaths = new Set<string>()
   for (const importer of Object.values(importers)) {
@@ -212,7 +247,7 @@ export async function writeLockfiles (
   }
 
   const forceSharedFormat = opts?.forceSharedFormat === true
-  const wantedLockfileToStringify = (opts.useInlineSpecifiersFormat ?? false)
+  const wantedLockfileToStringify = (Boolean(opts.useInlineSpecifiersFormat) || opts.wantedLockfile.lockfileVersion.toString().startsWith('6.'))
     ? convertToInlineSpecifiersFormat(opts.wantedLockfile) as unknown as Lockfile
     : opts.wantedLockfile
   const normalizeOpts = {