fix: reduce the side-effects cache key length

.changeset/late-points-lick.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/crypto.object-hasher": major
+---
+
+Initial release.

.changeset/rude-terms-begin.md

@@ -0,0 +1,6 @@
+---
+"@pnpm/calc-dep-state": major
+"pnpm": patch
+---
+
+Reduce the length of the side-effects cache key. Instead of saving a stringified object composed from the dependency versions of the package, use the hash calculated from the said object [#7563](https://github.com/pnpm/pnpm/pull/7563).

crypto/object-hasher/README.md

@@ -0,0 +1,13 @@
+# @pnpm/crypto.object-hasher
+
+> Generate hashes from objects
+
+## Installation
+
+```sh
+pnpm add @pnpm/crypto.object-hasher
+```
+
+## License
+
+MIT

crypto/object-hasher/jest.config.js

@@ -0,0 +1,3 @@
+const config = require('../../jest.config.js');
+
+module.exports = Object.assign({}, config, {});

crypto/object-hasher/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@pnpm/crypto.object-hasher",
+  "version": "0.0.0",
+  "description": "Generate hashes from objects",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "files": [
+    "lib",
+    "!*.map"
+  ],
+  "scripts": {
+    "lint": "eslint \"src/**/*.ts\" \"test/**/*.ts\"",
+    "_test": "jest",
+    "test": "pnpm run compile && pnpm run _test",
+    "prepublishOnly": "pnpm run compile",
+    "compile": "tsc --build && pnpm run lint --fix"
+  },
+  "repository": "https://github.com/pnpm/pnpm/blob/main/crypto/object-hasher",
+  "keywords": [
+    "pnpm9",
+    "hash",
+    "crypto"
+  ],
+  "engines": {
+    "node": ">=18.12"
+  },
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/pnpm/pnpm/issues"
+  },
+  "homepage": "https://github.com/pnpm/pnpm/blob/main/crypto/object-hasher#readme",
+  "dependencies": {
+    "node-object-hash": "3.0.0"
+  },
+  "devDependencies": {
+    "@pnpm/crypto.object-hasher": "workspace:*"
+  },
+  "funding": "https://opencollective.com/pnpm",
+  "exports": {
+    ".": "./lib/index.js"
+  }
+}

crypto/object-hasher/src/index.ts

@@ -0,0 +1,4 @@
+import { hasher } from 'node-object-hash'
+
+export const hashObjectWithoutSorting = hasher({ sort: false }).hash
+export const hashObject = hasher({}).hash

crypto/object-hasher/test/index.ts

@@ -0,0 +1,13 @@
+import { hashObject, hashObjectWithoutSorting } from '@pnpm/crypto.object-hasher'
+
+describe('hashObject', () => {
+  it('creates a hash', () => {
+    expect(hashObject({ b: 1, a: 2 })).toEqual('c8c943f9321eb7f98834b58391eee848d458c7b35211fc4911cdb1bbd877b74a')
+  })
+})
+
+describe('hashObjectWithoutSorting', () => {
+  it('creates a hash', () => {
+    expect(hashObjectWithoutSorting({ b: 1, a: 2 })).toEqual('c0a68b14aa1886a799c2e7c1289b65ccb79a668881d5b6956f0b185a9ac112d7')
+  })
+})

crypto/object-hasher/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "extends": "@pnpm/tsconfig",
+  "compilerOptions": {
+    "outDir": "lib",
+    "rootDir": "src"
+  },
+  "include": [
+    "src/**/*.ts",
+    "../../__typings__/**/*.d.ts"
+  ],
+  "references": [],
+  "composite": true
+}

crypto/object-hasher/tsconfig.lint.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig.json",
+  "include": [
+    "src/**/*.ts",
+    "test/**/*.ts",
+    "../../__typings__/**/*.d.ts"
+  ]
+}

exec/plugin-commands-rebuild/package.json

@@ -31,6 +31,7 @@
   "homepage": "https://github.com/pnpm/pnpm/blob/main/exec/plugin-commands-rebuild#readme",
   "devDependencies": {
     "@pnpm/assert-project": "workspace:*",
+    "@pnpm/crypto.object-hasher": "workspace:*",
     "@pnpm/filter-workspace-packages": "workspace:*",
     "@pnpm/plugin-commands-rebuild": "workspace:*",
     "@pnpm/prepare": "workspace:*",

exec/plugin-commands-rebuild/test/index.ts

@@ -3,6 +3,7 @@ import fs from 'fs'
 import path from 'path'
 import { getFilePathInCafs } from '@pnpm/store.cafs'
 import { ENGINE_NAME, WANTED_LOCKFILE } from '@pnpm/constants'
+import { hashObject } from '@pnpm/crypto.object-hasher'
 import { rebuild } from '@pnpm/plugin-commands-rebuild'
 import { prepare } from '@pnpm/prepare'
 import { getIntegrity, REGISTRY_MOCK_PORT } from '@pnpm/registry-mock'
@@ -78,7 +79,7 @@ test('rebuilds dependencies', async () => {
   const cacheIntegrityPath = getFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), 'index')
   const cacheIntegrity = await loadJsonFile<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any
   expect(cacheIntegrity!.sideEffects).toBeTruthy()
-  const sideEffectsKey = `${ENGINE_NAME}-${JSON.stringify({ '/@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}`
+  const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '/@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}`
   expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'generated-by-postinstall.js'])
   delete cacheIntegrity!.sideEffects[sideEffectsKey]['generated-by-postinstall.js']
 })
@@ -102,7 +103,7 @@ test('skipIfHasSideEffectsCache', async () => {
   const cafsDir = path.join(storeDir, 'v3/files')
   const cacheIntegrityPath = getFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), 'index')
   let cacheIntegrity = await loadJsonFile<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any
-  const sideEffectsKey = `${ENGINE_NAME}-${JSON.stringify({ '/@pnpm.e2e/hello-world-js-bin/1.0.0': {} })}`
+  const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '/@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}`
   cacheIntegrity.sideEffects = {
     [sideEffectsKey]: { foo: 'bar' },
   }

exec/plugin-commands-rebuild/tsconfig.json

@@ -33,6 +33,9 @@
     {
       "path": "../../config/normalize-registries"
     },
+    {
+      "path": "../../crypto/object-hasher"
+    },
     {
       "path": "../../deps/graph-sequencer"
     },

packages/calc-dep-state/package.json

@@ -30,6 +30,7 @@
   "homepage": "https://github.com/pnpm/pnpm/blob/main/packages/calc-dep-state#readme",
   "dependencies": {
     "@pnpm/constants": "workspace:*",
+    "@pnpm/crypto.object-hasher": "workspace:*",
     "@pnpm/dependency-path": "workspace:*",
     "@pnpm/lockfile-types": "workspace:*",
     "sort-keys": "^4.2.0"

packages/calc-dep-state/src/index.ts

@@ -1,6 +1,7 @@
 import { ENGINE_NAME } from '@pnpm/constants'
 import { refToRelative } from '@pnpm/dependency-path'
 import { type Lockfile } from '@pnpm/lockfile-types'
+import { hashObjectWithoutSorting } from '@pnpm/crypto.object-hasher'
 import sortKeys from 'sort-keys'
 
 export interface DepsGraph {
@@ -32,7 +33,7 @@ export function calcDepState (
   let result = ENGINE_NAME
   if (opts.isBuilt) {
     const depStateObj = calcDepStateObj(depPath, depsGraph, cache, new Set())
-    result += `-${JSON.stringify(depStateObj)}`
+    result += `-${hashObjectWithoutSorting(depStateObj)}`
   }
   if (opts.patchFileHash) {
     result += `-${opts.patchFileHash}`

packages/calc-dep-state/test/index.ts

@@ -1,5 +1,6 @@
 import { calcDepState } from '@pnpm/calc-dep-state'
 import { ENGINE_NAME } from '@pnpm/constants'
+import { hashObject } from '@pnpm/crypto.object-hasher'
 
 const depsGraph = {
   'registry/foo@1.0.0': {
@@ -19,7 +20,7 @@ const depsGraph = {
 test('calcDepState()', () => {
   expect(calcDepState(depsGraph, {}, '/registry/foo@1.0.0', {
     isBuilt: true,
-  })).toBe(`${ENGINE_NAME}-{}`)
+  })).toBe(`${ENGINE_NAME}-${hashObject({})}`)
 })
 
 test('calcDepState() when scripts are ignored', () => {

packages/calc-dep-state/tsconfig.json

@@ -9,6 +9,9 @@
     "../../__typings__/**/*.d.ts"
   ],
   "references": [
+    {
+      "path": "../../crypto/object-hasher"
+    },
     {
       "path": "../../lockfile/lockfile-types"
     },

pkg-manager/core/package.json

@@ -77,6 +77,7 @@
     "@pnpm/assert-store": "workspace:*",
     "@pnpm/client": "workspace:*",
     "@pnpm/core": "workspace:*",
+    "@pnpm/crypto.object-hasher": "workspace:*",
     "@pnpm/git-utils": "workspace:*",
     "@pnpm/lockfile-types": "workspace:*",
     "@pnpm/package-store": "workspace:*",

pkg-manager/core/test/install/sideEffects.ts

@@ -1,6 +1,7 @@
 import { promises as fs, existsSync, readFileSync } from 'fs'
 import path from 'path'
 import { addDependenciesToPackage, install } from '@pnpm/core'
+import { hashObject } from '@pnpm/crypto.object-hasher'
 import { getFilePathInCafs, getFilePathByModeInCafs, type PackageFilesIndex } from '@pnpm/store.cafs'
 import { getIntegrity, REGISTRY_MOCK_PORT } from '@pnpm/registry-mock'
 import { prepareEmpty } from '@pnpm/prepare'
@@ -86,7 +87,7 @@ test('using side effects cache', async () => {
   const filesIndexFile = getFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), 'index')
   const filesIndex = await loadJsonFile<PackageFilesIndex>(filesIndexFile)
   expect(filesIndex.sideEffects).toBeTruthy() // files index has side effects
-  const sideEffectsKey = `${ENGINE_NAME}-${JSON.stringify({ '/@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}`
+  const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '/@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}`
   expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'generated-by-preinstall.js'])
   expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'generated-by-postinstall.js'])
   delete filesIndex.sideEffects![sideEffectsKey]['generated-by-postinstall.js']
@@ -177,7 +178,7 @@ test('a postinstall script does not modify the original sources added to the sto
   const cafsDir = path.join(opts.storeDir, 'files')
   const filesIndexFile = getFilePathInCafs(cafsDir, getIntegrity('@pnpm/postinstall-modifies-source', '1.0.0'), 'index')
   const filesIndex = await loadJsonFile<PackageFilesIndex>(filesIndexFile)
-  const patchedFileIntegrity = filesIndex.sideEffects?.[`${ENGINE_NAME}-{}`]['empty-file.txt']?.integrity
+  const patchedFileIntegrity = filesIndex.sideEffects?.[`${ENGINE_NAME}-${hashObject({})}`]['empty-file.txt']?.integrity
   expect(patchedFileIntegrity).toBeTruthy()
   const originalFileIntegrity = filesIndex.files['empty-file.txt'].integrity
   expect(originalFileIntegrity).toBeTruthy()
@@ -201,7 +202,7 @@ test('a corrupted side-effects cache is ignored', async () => {
   const filesIndexFile = getFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), 'index')
   const filesIndex = await loadJsonFile<PackageFilesIndex>(filesIndexFile)
   expect(filesIndex.sideEffects).toBeTruthy() // files index has side effects
-  const sideEffectsKey = `${ENGINE_NAME}-${JSON.stringify({ '/@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}`
+  const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '/@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}`
   expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'generated-by-preinstall.js'])
   const sideEffectFileStat = filesIndex.sideEffects![sideEffectsKey]['generated-by-preinstall.js']
   const sideEffectFile = getFilePathByModeInCafs(cafsDir, sideEffectFileStat.integrity, sideEffectFileStat.mode)

pkg-manager/core/tsconfig.json

@@ -30,6 +30,9 @@
     {
       "path": "../../config/normalize-registries"
     },
+    {
+      "path": "../../crypto/object-hasher"
+    },
     {
       "path": "../../deps/graph-sequencer"
     },

pkg-manager/headless/package.json

@@ -18,6 +18,7 @@
   "devDependencies": {
     "@pnpm/assert-project": "workspace:*",
     "@pnpm/client": "workspace:*",
+    "@pnpm/crypto.object-hasher": "workspace:*",
     "@pnpm/headless": "workspace:*",
     "@pnpm/package-store": "workspace:*",
     "@pnpm/prepare": "workspace:*",

pkg-manager/headless/test/index.ts

@@ -2,6 +2,7 @@
 import { promises as fs, existsSync, realpathSync, writeFileSync } from 'fs'
 import path from 'path'
 import { assertProject } from '@pnpm/assert-project'
+import { hashObject } from '@pnpm/crypto.object-hasher'
 import { getFilePathInCafs } from '@pnpm/store.cafs'
 import { ENGINE_NAME, WANTED_LOCKFILE } from '@pnpm/constants'
 import {
@@ -685,7 +686,7 @@ test.each([['isolated'], ['hoisted']])('using side effects cache with nodeLinker
   const cacheIntegrityPath = getFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), 'index')
   const cacheIntegrity = await loadJsonFile<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any
   expect(cacheIntegrity!.sideEffects).toBeTruthy()
-  const sideEffectsKey = `${ENGINE_NAME}-${JSON.stringify({ '/@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}`
+  const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '/@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}`
   expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'generated-by-postinstall.js'])
   delete cacheIntegrity!.sideEffects[sideEffectsKey]['generated-by-postinstall.js']
 

pkg-manager/headless/tsconfig.json

@@ -24,6 +24,9 @@
     {
       "path": "../../config/package-is-installable"
     },
+    {
+      "path": "../../crypto/object-hasher"
+    },
     {
       "path": "../../deps/graph-builder"
     },

pnpm-workspace.yaml

@@ -5,6 +5,7 @@ packages:
   - "!__utils__/build-artifacts"
   - cli/*
   - config/*
+  - crypto/*
   - dedupe/*
   - deps/*
   - env/*