From 9ffa5247014325916bf031d7bb761d7a3bc0abc7 Mon Sep 17 00:00:00 2001 From: Matthias Junker Date: Tue, 18 Apr 2023 11:47:58 +0200 Subject: [PATCH 01/14] feat: support lockfileVersion 3 --- lib/modules/manager/npm/extract/locked-versions.ts | 7 +++++-- lib/modules/manager/npm/extract/npm.ts | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/modules/manager/npm/extract/locked-versions.ts b/lib/modules/manager/npm/extract/locked-versions.ts index dde7884b90a31f..5fcc0a5db84864 100644 --- a/lib/modules/manager/npm/extract/locked-versions.ts +++ b/lib/modules/manager/npm/extract/locked-versions.ts @@ -69,7 +69,7 @@ export async function getLockedVersions( } else { packageFile.extractedConstraints!.npm = '<7'; } - } else if (lockfileVersion === 2) { + } else if (lockfileVersion === 2 || lockfileVersion === 3) { if (packageFile.extractedConstraints?.npm) { // Add a <9 constraint if the latest 8.x is compatible if ( @@ -84,7 +84,10 @@ export async function getLockedVersions( for (const dep of packageFile.deps) { // TODO: types (#7154) dep.lockedVersion = semver.valid( - lockFileCache[npmLock].lockedVersions[dep.depName!] + lockFileCache[npmLock].lockedVersions[dep.depName!] || + lockFileCache[npmLock].lockedVersions[ + `node_modules/${dep.depName!}` + ] )!; } } else if (pnpmShrinkwrap) { diff --git a/lib/modules/manager/npm/extract/npm.ts b/lib/modules/manager/npm/extract/npm.ts index 7a756734b92ef3..fe431af81403b4 100644 --- a/lib/modules/manager/npm/extract/npm.ts +++ b/lib/modules/manager/npm/extract/npm.ts @@ -9,7 +9,7 @@ export async function getNpmLock(filePath: string): Promise { const lockParsed = JSON.parse(lockRaw); const lockedVersions: Record = {}; for (const [entry, val] of Object.entries( - (lockParsed.dependencies || {}) as LockFileEntry + (lockParsed.dependencies || lockParsed.packages || {}) as LockFileEntry )) { logger.trace({ entry, version: val.version }); lockedVersions[entry] = val.version; From 5108bb1865a767bbe6fbf4b91af4cc8761c1a1ba Mon Sep 17 00:00:00 2001 From: Matthias Junker Date: Wed, 17 May 2023 08:55:57 +0200 Subject: [PATCH 02/14] refactor: extract dependencies and added tests --- .../npm/__fixtures__/npm9/package-lock.json | 80 +++++++++ .../npm/__fixtures__/npm9/package.json | 15 ++ .../extract/__snapshots__/npm.spec.ts.snap | 15 ++ .../npm/extract/locked-versions.spec.ts | 159 +++++++++++++++--- .../manager/npm/extract/locked-versions.ts | 12 +- lib/modules/manager/npm/extract/npm.spec.ts | 11 ++ lib/modules/manager/npm/extract/npm.ts | 21 ++- 7 files changed, 282 insertions(+), 31 deletions(-) create mode 100644 lib/modules/manager/npm/__fixtures__/npm9/package-lock.json create mode 100644 lib/modules/manager/npm/__fixtures__/npm9/package.json diff --git a/lib/modules/manager/npm/__fixtures__/npm9/package-lock.json b/lib/modules/manager/npm/__fixtures__/npm9/package-lock.json new file mode 100644 index 00000000000000..d637c965ca07ed --- /dev/null +++ b/lib/modules/manager/npm/__fixtures__/npm9/package-lock.json @@ -0,0 +1,80 @@ +{ + "name": "npm9", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "npm9", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "chalk": "^2.4.1" + } + }, + "node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" + }, + "node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "engines": { + "node": ">=4" + } + }, + "node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + } + } +} diff --git a/lib/modules/manager/npm/__fixtures__/npm9/package.json b/lib/modules/manager/npm/__fixtures__/npm9/package.json new file mode 100644 index 00000000000000..54614b8a2649fe --- /dev/null +++ b/lib/modules/manager/npm/__fixtures__/npm9/package.json @@ -0,0 +1,15 @@ +{ + "name": "npm9", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "chalk": "^2.4.1" + } +} diff --git a/lib/modules/manager/npm/extract/__snapshots__/npm.spec.ts.snap b/lib/modules/manager/npm/extract/__snapshots__/npm.spec.ts.snap index 91caae831ab088..8d769febff3b35 100644 --- a/lib/modules/manager/npm/extract/__snapshots__/npm.spec.ts.snap +++ b/lib/modules/manager/npm/extract/__snapshots__/npm.spec.ts.snap @@ -29,3 +29,18 @@ exports[`modules/manager/npm/extract/npm .getNpmLock() extracts npm 7 lockfile 1 "lockfileVersion": 2, } `; + +exports[`modules/manager/npm/extract/npm .getNpmLock() extracts npm 9 lockfile 1`] = ` +{ + "lockedVersions": { + "ansi-styles": "3.2.1", + "chalk": "2.4.2", + "color-convert": "1.9.3", + "color-name": "1.1.3", + "escape-string-regexp": "1.0.5", + "has-flag": "3.0.0", + "supports-color": "5.5.0", + }, + "lockfileVersion": 3, +} +`; diff --git a/lib/modules/manager/npm/extract/locked-versions.spec.ts b/lib/modules/manager/npm/extract/locked-versions.spec.ts index 6f411397e710f0..3de286b40b37a9 100644 --- a/lib/modules/manager/npm/extract/locked-versions.spec.ts +++ b/lib/modules/manager/npm/extract/locked-versions.spec.ts @@ -490,49 +490,162 @@ describe('modules/manager/npm/extract/locked-versions', () => { }); }); - it('uses pnpm-lock', async () => { - pnpm.getPnpmLock.mockReturnValue({ - lockedVersions: { - a: '1.0.0', - b: '2.0.0', - c: '3.0.0', - }, - lockfileVersion: 6.0, + describe('lockfileVersion 3', () => { + it('uses package-lock.json with npm v9.0.0', async () => { + npm.getNpmLock.mockReturnValue({ + lockedVersions: { + a: '1.0.0', + b: '2.0.0', + c: '3.0.0', + }, + lockfileVersion: 3, + }); + const packageFiles = [ + { + managerData: { + npmLock: 'package-lock.json', + }, + extractedConstraints: {}, + deps: [ + { depName: 'a', currentValue: '1.0.0' }, + { depName: 'b', currentValue: '2.0.0' }, + ], + packageFile: 'some-file', + }, + ]; + await getLockedVersions(packageFiles); + expect(packageFiles).toEqual([ + { + extractedConstraints: { + npm: '>=7', + }, + deps: [ + { currentValue: '1.0.0', depName: 'a', lockedVersion: '1.0.0' }, + { currentValue: '2.0.0', depName: 'b', lockedVersion: '2.0.0' }, + ], + packageFile: 'some-file', + lockFiles: ['package-lock.json'], + managerData: { + npmLock: 'package-lock.json', + }, + }, + ]); }); + + it('uses package-lock.json with npm v7.0.0', async () => { + npm.getNpmLock.mockReturnValue({ + lockedVersions: { + a: '1.0.0', + b: '2.0.0', + c: '3.0.0', + }, + lockfileVersion: 3, + }); + const packageFiles = [ + { + managerData: { + npmLock: 'package-lock.json', + }, + extractedConstraints: { + npm: '^9.0.0', + }, + deps: [ + { depName: 'a', currentValue: '1.0.0' }, + { depName: 'b', currentValue: '2.0.0' }, + ], + packageFile: 'some-file', + }, + ]; + await getLockedVersions(packageFiles); + expect(packageFiles).toEqual([ + { + extractedConstraints: { + npm: '^9.0.0', + }, + deps: [ + { currentValue: '1.0.0', depName: 'a', lockedVersion: '1.0.0' }, + { currentValue: '2.0.0', depName: 'b', lockedVersion: '2.0.0' }, + ], + packageFile: 'some-file', + lockFiles: ['package-lock.json'], + managerData: { + npmLock: 'package-lock.json', + }, + }, + ]); + }); + }); + + it('ignores pnpm', async () => { const packageFiles = [ { managerData: { pnpmShrinkwrap: 'pnpm-lock.yaml', }, - extractedConstraints: { - pnpm: '>=6.0.0', - }, deps: [ - { - depName: 'a', - currentValue: '1.0.0', - }, - { - depName: 'b', - currentValue: '2.0.0', - }, + { depName: 'a', currentValue: '1.0.0' }, + { depName: 'b', currentValue: '2.0.0' }, ], packageFile: 'some-file', }, ]; - pnpm.getConstraints.mockReturnValue('>=6.0.0 >=8'); await getLockedVersions(packageFiles); expect(packageFiles).toEqual([ { - extractedConstraints: { pnpm: '>=6.0.0 >=8' }, deps: [ - { currentValue: '1.0.0', depName: 'a', lockedVersion: '1.0.0' }, - { currentValue: '2.0.0', depName: 'b', lockedVersion: '2.0.0' }, + { currentValue: '1.0.0', depName: 'a' }, + { currentValue: '2.0.0', depName: 'b' }, ], lockFiles: ['pnpm-lock.yaml'], managerData: { pnpmShrinkwrap: 'pnpm-lock.yaml' }, packageFile: 'some-file', }, ]); + + it('uses pnpm-lock', async () => { + pnpm.getPnpmLock.mockReturnValue({ + lockedVersions: { + a: '1.0.0', + b: '2.0.0', + c: '3.0.0', + }, + lockfileVersion: 6.0, + }); + const packageFiles = [ + { + managerData: { + pnpmShrinkwrap: 'pnpm-lock.yaml', + }, + extractedConstraints: { + pnpm: '>=6.0.0', + }, + deps: [ + { + depName: 'a', + currentValue: '1.0.0', + }, + { + depName: 'b', + currentValue: '2.0.0', + }, + ], + packageFile: 'some-file', + }, + ]; + pnpm.getConstraints.mockReturnValue('>=6.0.0 >=8'); + await getLockedVersions(packageFiles); + expect(packageFiles).toEqual([ + { + extractedConstraints: { pnpm: '>=6.0.0 >=8' }, + deps: [ + { currentValue: '1.0.0', depName: 'a', lockedVersion: '1.0.0' }, + { currentValue: '2.0.0', depName: 'b', lockedVersion: '2.0.0' }, + ], + lockFiles: ['pnpm-lock.yaml'], + managerData: { pnpmShrinkwrap: 'pnpm-lock.yaml' }, + packageFile: 'some-file', + }, + ]); + }); }); }); diff --git a/lib/modules/manager/npm/extract/locked-versions.ts b/lib/modules/manager/npm/extract/locked-versions.ts index 5fcc0a5db84864..8b7c4dae1c03ea 100644 --- a/lib/modules/manager/npm/extract/locked-versions.ts +++ b/lib/modules/manager/npm/extract/locked-versions.ts @@ -69,7 +69,7 @@ export async function getLockedVersions( } else { packageFile.extractedConstraints!.npm = '<7'; } - } else if (lockfileVersion === 2 || lockfileVersion === 3) { + } else if (lockfileVersion === 2) { if (packageFile.extractedConstraints?.npm) { // Add a <9 constraint if the latest 8.x is compatible if ( @@ -80,14 +80,16 @@ export async function getLockedVersions( } else { packageFile.extractedConstraints!.npm = '<9'; } + } else if ( + lockfileVersion === 3 && + !packageFile.extractedConstraints?.npm + ) { + packageFile.extractedConstraints!.npm = '>=7'; } for (const dep of packageFile.deps) { // TODO: types (#7154) dep.lockedVersion = semver.valid( - lockFileCache[npmLock].lockedVersions[dep.depName!] || - lockFileCache[npmLock].lockedVersions[ - `node_modules/${dep.depName!}` - ] + lockFileCache[npmLock].lockedVersions[dep.depName!] )!; } } else if (pnpmShrinkwrap) { diff --git a/lib/modules/manager/npm/extract/npm.spec.ts b/lib/modules/manager/npm/extract/npm.spec.ts index 321b56501dd0d7..93c07ee39f8495 100644 --- a/lib/modules/manager/npm/extract/npm.spec.ts +++ b/lib/modules/manager/npm/extract/npm.spec.ts @@ -18,6 +18,7 @@ describe('modules/manager/npm/extract/npm', () => { const res = await getNpmLock('package.json'); expect(res).toMatchSnapshot(); expect(Object.keys(res.lockedVersions)).toHaveLength(7); + expect(res.lockfileVersion).toBe(1); }); it('extracts npm 7 lockfile', async () => { @@ -29,6 +30,16 @@ describe('modules/manager/npm/extract/npm', () => { expect(res.lockfileVersion).toBe(2); }); + it('extracts npm 9 lockfile', async () => { + const npm9Lock = Fixtures.get('npm9/package-lock.json', '..'); + fs.readLocalFile.mockResolvedValueOnce(npm9Lock as never); + const res = await getNpmLock('package.json'); + expect(res).toMatchSnapshot(); + + expect(Object.keys(res.lockedVersions)).toHaveLength(7); + expect(res.lockfileVersion).toBe(3); + }); + it('returns empty if no deps', async () => { fs.readLocalFile.mockResolvedValueOnce('{}'); const res = await getNpmLock('package.json'); diff --git a/lib/modules/manager/npm/extract/npm.ts b/lib/modules/manager/npm/extract/npm.ts index fe431af81403b4..bdcf4a9626b55d 100644 --- a/lib/modules/manager/npm/extract/npm.ts +++ b/lib/modules/manager/npm/extract/npm.ts @@ -8,9 +8,7 @@ export async function getNpmLock(filePath: string): Promise { try { const lockParsed = JSON.parse(lockRaw); const lockedVersions: Record = {}; - for (const [entry, val] of Object.entries( - (lockParsed.dependencies || lockParsed.packages || {}) as LockFileEntry - )) { + for (const [entry, val] of Object.entries(getPackages(lockParsed))) { logger.trace({ entry, version: val.version }); lockedVersions[entry] = val.version; } @@ -20,3 +18,20 @@ export async function getNpmLock(filePath: string): Promise { return { lockedVersions: {} }; } } +function getPackages(lockParsed: any): LockFileEntry { + let packages: LockFileEntry = {}; + + if ( + (lockParsed.lockfileVersion === 1 || lockParsed.lockfileVersion === 2) && + lockParsed.dependencies + ) { + packages = lockParsed.dependencies; + } else if (lockParsed.lockfileVersion === 3 && lockParsed.packages) { + packages = Object.fromEntries( + Object.entries(lockParsed.packages) + .filter(([key]) => !!key) // filter out root entry + .map(([key, val]) => [key.replace(`node_modules/`, ''), val]) + ) as LockFileEntry; + } + return packages; +} From 7a69f6957e8c1932cf68366ef8a11433f4ad17f4 Mon Sep 17 00:00:00 2001 From: Matthias Junker Date: Wed, 17 May 2023 14:24:34 +0200 Subject: [PATCH 03/14] chore: fix review findings --- .../extract/__snapshots__/npm.spec.ts.snap | 46 ------- .../npm/extract/locked-versions.spec.ts | 119 +++++++----------- lib/modules/manager/npm/extract/npm.spec.ts | 42 ++++++- 3 files changed, 82 insertions(+), 125 deletions(-) delete mode 100644 lib/modules/manager/npm/extract/__snapshots__/npm.spec.ts.snap diff --git a/lib/modules/manager/npm/extract/__snapshots__/npm.spec.ts.snap b/lib/modules/manager/npm/extract/__snapshots__/npm.spec.ts.snap deleted file mode 100644 index 8d769febff3b35..00000000000000 --- a/lib/modules/manager/npm/extract/__snapshots__/npm.spec.ts.snap +++ /dev/null @@ -1,46 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`modules/manager/npm/extract/npm .getNpmLock() extracts 1`] = ` -{ - "lockedVersions": { - "ansi-styles": "3.2.1", - "chalk": "2.4.1", - "color-convert": "1.9.1", - "color-name": "1.1.3", - "escape-string-regexp": "1.0.5", - "has-flag": "3.0.0", - "supports-color": "5.4.0", - }, - "lockfileVersion": 1, -} -`; - -exports[`modules/manager/npm/extract/npm .getNpmLock() extracts npm 7 lockfile 1`] = ` -{ - "lockedVersions": { - "ansi-styles": "3.2.1", - "chalk": "2.4.1", - "color-convert": "1.9.1", - "color-name": "1.1.3", - "escape-string-regexp": "1.0.5", - "has-flag": "3.0.0", - "supports-color": "5.4.0", - }, - "lockfileVersion": 2, -} -`; - -exports[`modules/manager/npm/extract/npm .getNpmLock() extracts npm 9 lockfile 1`] = ` -{ - "lockedVersions": { - "ansi-styles": "3.2.1", - "chalk": "2.4.2", - "color-convert": "1.9.3", - "color-name": "1.1.3", - "escape-string-regexp": "1.0.5", - "has-flag": "3.0.0", - "supports-color": "5.5.0", - }, - "lockfileVersion": 3, -} -`; diff --git a/lib/modules/manager/npm/extract/locked-versions.spec.ts b/lib/modules/manager/npm/extract/locked-versions.spec.ts index 3de286b40b37a9..252c3fa94733d5 100644 --- a/lib/modules/manager/npm/extract/locked-versions.spec.ts +++ b/lib/modules/manager/npm/extract/locked-versions.spec.ts @@ -490,6 +490,52 @@ describe('modules/manager/npm/extract/locked-versions', () => { }); }); + it('uses pnpm-lock', async () => { + pnpm.getPnpmLock.mockReturnValue({ + lockedVersions: { + a: '1.0.0', + b: '2.0.0', + c: '3.0.0', + }, + lockfileVersion: 6.0, + }); + const packageFiles = [ + { + managerData: { + pnpmShrinkwrap: 'pnpm-lock.yaml', + }, + extractedConstraints: { + pnpm: '>=6.0.0', + }, + deps: [ + { + depName: 'a', + currentValue: '1.0.0', + }, + { + depName: 'b', + currentValue: '2.0.0', + }, + ], + packageFile: 'some-file', + }, + ]; + pnpm.getConstraints.mockReturnValue('>=6.0.0 >=8'); + await getLockedVersions(packageFiles); + expect(packageFiles).toEqual([ + { + extractedConstraints: { pnpm: '>=6.0.0 >=8' }, + deps: [ + { currentValue: '1.0.0', depName: 'a', lockedVersion: '1.0.0' }, + { currentValue: '2.0.0', depName: 'b', lockedVersion: '2.0.0' }, + ], + lockFiles: ['pnpm-lock.yaml'], + managerData: { pnpmShrinkwrap: 'pnpm-lock.yaml' }, + packageFile: 'some-file', + }, + ]); + }); + describe('lockfileVersion 3', () => { it('uses package-lock.json with npm v9.0.0', async () => { npm.getNpmLock.mockReturnValue({ @@ -575,77 +621,4 @@ describe('modules/manager/npm/extract/locked-versions', () => { ]); }); }); - - it('ignores pnpm', async () => { - const packageFiles = [ - { - managerData: { - pnpmShrinkwrap: 'pnpm-lock.yaml', - }, - deps: [ - { depName: 'a', currentValue: '1.0.0' }, - { depName: 'b', currentValue: '2.0.0' }, - ], - packageFile: 'some-file', - }, - ]; - await getLockedVersions(packageFiles); - expect(packageFiles).toEqual([ - { - deps: [ - { currentValue: '1.0.0', depName: 'a' }, - { currentValue: '2.0.0', depName: 'b' }, - ], - lockFiles: ['pnpm-lock.yaml'], - managerData: { pnpmShrinkwrap: 'pnpm-lock.yaml' }, - packageFile: 'some-file', - }, - ]); - - it('uses pnpm-lock', async () => { - pnpm.getPnpmLock.mockReturnValue({ - lockedVersions: { - a: '1.0.0', - b: '2.0.0', - c: '3.0.0', - }, - lockfileVersion: 6.0, - }); - const packageFiles = [ - { - managerData: { - pnpmShrinkwrap: 'pnpm-lock.yaml', - }, - extractedConstraints: { - pnpm: '>=6.0.0', - }, - deps: [ - { - depName: 'a', - currentValue: '1.0.0', - }, - { - depName: 'b', - currentValue: '2.0.0', - }, - ], - packageFile: 'some-file', - }, - ]; - pnpm.getConstraints.mockReturnValue('>=6.0.0 >=8'); - await getLockedVersions(packageFiles); - expect(packageFiles).toEqual([ - { - extractedConstraints: { pnpm: '>=6.0.0 >=8' }, - deps: [ - { currentValue: '1.0.0', depName: 'a', lockedVersion: '1.0.0' }, - { currentValue: '2.0.0', depName: 'b', lockedVersion: '2.0.0' }, - ], - lockFiles: ['pnpm-lock.yaml'], - managerData: { pnpmShrinkwrap: 'pnpm-lock.yaml' }, - packageFile: 'some-file', - }, - ]); - }); - }); }); diff --git a/lib/modules/manager/npm/extract/npm.spec.ts b/lib/modules/manager/npm/extract/npm.spec.ts index 93c07ee39f8495..ce6d8311cd6549 100644 --- a/lib/modules/manager/npm/extract/npm.spec.ts +++ b/lib/modules/manager/npm/extract/npm.spec.ts @@ -16,7 +16,18 @@ describe('modules/manager/npm/extract/npm', () => { const plocktest1Lock = Fixtures.get('plocktest1/package-lock.json', '..'); fs.readLocalFile.mockResolvedValueOnce(plocktest1Lock as never); const res = await getNpmLock('package.json'); - expect(res).toMatchSnapshot(); + expect(res).toEqual({ + lockedVersions: { + 'ansi-styles': '3.2.1', + chalk: '2.4.1', + 'color-convert': '1.9.1', + 'color-name': '1.1.3', + 'escape-string-regexp': '1.0.5', + 'has-flag': '3.0.0', + 'supports-color': '5.4.0', + }, + lockfileVersion: 1, + }); expect(Object.keys(res.lockedVersions)).toHaveLength(7); expect(res.lockfileVersion).toBe(1); }); @@ -25,7 +36,18 @@ describe('modules/manager/npm/extract/npm', () => { const npm7Lock = Fixtures.get('npm7/package-lock.json', '..'); fs.readLocalFile.mockResolvedValueOnce(npm7Lock as never); const res = await getNpmLock('package.json'); - expect(res).toMatchSnapshot(); + expect(res).toEqual({ + lockedVersions: { + 'ansi-styles': '3.2.1', + chalk: '2.4.1', + 'color-convert': '1.9.1', + 'color-name': '1.1.3', + 'escape-string-regexp': '1.0.5', + 'has-flag': '3.0.0', + 'supports-color': '5.4.0', + }, + lockfileVersion: 2, + }); expect(Object.keys(res.lockedVersions)).toHaveLength(7); expect(res.lockfileVersion).toBe(2); }); @@ -34,10 +56,18 @@ describe('modules/manager/npm/extract/npm', () => { const npm9Lock = Fixtures.get('npm9/package-lock.json', '..'); fs.readLocalFile.mockResolvedValueOnce(npm9Lock as never); const res = await getNpmLock('package.json'); - expect(res).toMatchSnapshot(); - - expect(Object.keys(res.lockedVersions)).toHaveLength(7); - expect(res.lockfileVersion).toBe(3); + expect(res).toEqual({ + lockedVersions: { + 'ansi-styles': '3.2.1', + chalk: '2.4.2', + 'color-convert': '1.9.3', + 'color-name': '1.1.3', + 'escape-string-regexp': '1.0.5', + 'has-flag': '3.0.0', + 'supports-color': '5.5.0', + }, + lockfileVersion: 3, + }); }); it('returns empty if no deps', async () => { From c6275d1d6eb3901ff0f276925e6a2b05da95ef04 Mon Sep 17 00:00:00 2001 From: Matthias Junker Date: Fri, 19 May 2023 11:00:59 +0200 Subject: [PATCH 04/14] feat: added schema for packagelock --- .../manager/npm/extract/locked-versions.ts | 6 ++ lib/modules/manager/npm/extract/npm.spec.ts | 58 +++++++++++++++++-- lib/modules/manager/npm/extract/npm.ts | 42 +++++++++----- lib/modules/manager/npm/extract/schema.ts | 36 ++++++++++++ 4 files changed, 121 insertions(+), 21 deletions(-) create mode 100644 lib/modules/manager/npm/extract/schema.ts diff --git a/lib/modules/manager/npm/extract/locked-versions.ts b/lib/modules/manager/npm/extract/locked-versions.ts index 8b7c4dae1c03ea..7246c8b1801195 100644 --- a/lib/modules/manager/npm/extract/locked-versions.ts +++ b/lib/modules/manager/npm/extract/locked-versions.ts @@ -85,6 +85,12 @@ export async function getLockedVersions( !packageFile.extractedConstraints?.npm ) { packageFile.extractedConstraints!.npm = '>=7'; + } else { + logger.warn( + { lockfileVersion }, + 'Found unsupported npm lockfile version' + ); + return; } for (const dep of packageFile.deps) { // TODO: types (#7154) diff --git a/lib/modules/manager/npm/extract/npm.spec.ts b/lib/modules/manager/npm/extract/npm.spec.ts index ce6d8311cd6549..238badaa780cab 100644 --- a/lib/modules/manager/npm/extract/npm.spec.ts +++ b/lib/modules/manager/npm/extract/npm.spec.ts @@ -1,6 +1,6 @@ import { Fixtures } from '../../../../../test/fixtures'; import { fs } from '../../../../../test/util'; -import { getNpmLock } from './npm'; +import { extractPackages, getNpmLock } from './npm'; jest.mock('../../../../util/fs'); @@ -28,8 +28,6 @@ describe('modules/manager/npm/extract/npm', () => { }, lockfileVersion: 1, }); - expect(Object.keys(res.lockedVersions)).toHaveLength(7); - expect(res.lockfileVersion).toBe(1); }); it('extracts npm 7 lockfile', async () => { @@ -48,8 +46,6 @@ describe('modules/manager/npm/extract/npm', () => { }, lockfileVersion: 2, }); - expect(Object.keys(res.lockedVersions)).toHaveLength(7); - expect(res.lockfileVersion).toBe(2); }); it('extracts npm 9 lockfile', async () => { @@ -76,4 +72,56 @@ describe('modules/manager/npm/extract/npm', () => { expect(Object.keys(res.lockedVersions)).toHaveLength(0); }); }); + + describe('.extractPackages', () => { + it('should parse lockfileVersion 1 without dependencies', () => { + expect( + extractPackages({ + name: 'no_dependencies', + version: '1.0.0', + lockfileVersion: 1, + }) + ).toEqual({ + packages: {}, + lockfileVersion: 1, + }); + }); + + it('should not throw if additional property exists', () => { + const npm9Lock = Fixtures.get('npm9/package-lock.json', '..'); + expect(() => + extractPackages({ ...JSON.parse(npm9Lock), additionalProperty: 'test' }) + ).not.toThrow(); + }); + + it('should throw if lockfileVersion is invalid', () => { + const npm9Lock = Fixtures.get('npm9/package-lock.json', '..'); + expect(() => { + extractPackages({ ...JSON.parse(npm9Lock), lockfileVersion: 4 }); + }).toThrow( + 'Invalid package-lock file. Neither v1, v2 nor v3 schema matched' + ); + }); + + it('should throw if lock file is empty', () => { + expect(() => { + extractPackages({}); + }).toThrow( + 'Invalid package-lock file. Neither v1, v2 nor v3 schema matched' + ); + }); + + it('should throw if lockfileVersion 3 without packages property', () => { + const npm9Lock = Fixtures.get('npm9/package-lock.json', '..'); + expect(() => { + extractPackages({ + ...JSON.parse(npm9Lock), + packages: undefined, + lockfileVersion: 3, + }); + }).toThrow( + 'Invalid package-lock file. Neither v1, v2 nor v3 schema matched' + ); + }); + }); }); diff --git a/lib/modules/manager/npm/extract/npm.ts b/lib/modules/manager/npm/extract/npm.ts index bdcf4a9626b55d..ac9c5086b7d88d 100644 --- a/lib/modules/manager/npm/extract/npm.ts +++ b/lib/modules/manager/npm/extract/npm.ts @@ -1,5 +1,6 @@ import { logger } from '../../../../logger'; import { readLocalFile } from '../../../../util/fs'; +import { PackageLockPreV3Schema, PackageLockV3Schema } from './schema'; import type { LockFile, LockFileEntry } from './types'; export async function getNpmLock(filePath: string): Promise { @@ -8,30 +9,39 @@ export async function getNpmLock(filePath: string): Promise { try { const lockParsed = JSON.parse(lockRaw); const lockedVersions: Record = {}; - for (const [entry, val] of Object.entries(getPackages(lockParsed))) { + const { packages, lockfileVersion } = extractPackages(lockParsed); + for (const [entry, val] of Object.entries(packages)) { logger.trace({ entry, version: val.version }); lockedVersions[entry] = val.version; } - return { lockedVersions, lockfileVersion: lockParsed.lockfileVersion }; + return { lockedVersions, lockfileVersion }; } catch (err) { logger.debug({ filePath, err }, 'Warning: Exception parsing npm lock file'); return { lockedVersions: {} }; } } -function getPackages(lockParsed: any): LockFileEntry { - let packages: LockFileEntry = {}; - if ( - (lockParsed.lockfileVersion === 1 || lockParsed.lockfileVersion === 2) && - lockParsed.dependencies - ) { - packages = lockParsed.dependencies; - } else if (lockParsed.lockfileVersion === 3 && lockParsed.packages) { - packages = Object.fromEntries( - Object.entries(lockParsed.packages) - .filter(([key]) => !!key) // filter out root entry - .map(([key, val]) => [key.replace(`node_modules/`, ''), val]) - ) as LockFileEntry; +export function extractPackages(lockParsed: any): { + packages: LockFileEntry; + lockfileVersion: number; +} { + const packageLockPreV3ParseResult = + PackageLockPreV3Schema.safeParse(lockParsed); + const packageLockV3ParseResult = PackageLockV3Schema.safeParse(lockParsed); + + if (packageLockPreV3ParseResult.success) { + return { + packages: packageLockPreV3ParseResult.data.dependencies ?? {}, + lockfileVersion: packageLockPreV3ParseResult.data.lockfileVersion, + }; + } else if (packageLockV3ParseResult.success) { + return { + packages: packageLockV3ParseResult.data.packages, + lockfileVersion: packageLockV3ParseResult.data.lockfileVersion, + }; + } else { + throw new Error( + 'Invalid package-lock file. Neither v1, v2 nor v3 schema matched' + ); } - return packages; } diff --git a/lib/modules/manager/npm/extract/schema.ts b/lib/modules/manager/npm/extract/schema.ts new file mode 100644 index 00000000000000..c95261588cf3b0 --- /dev/null +++ b/lib/modules/manager/npm/extract/schema.ts @@ -0,0 +1,36 @@ +import { z } from 'zod'; + +/** + * Removes the entry with an empty key which is used + * in packagelock v3 to indicate a root package. + */ +const removeRecordsWithEmptyKeys = (value: any): any => + Object.fromEntries( + Object.entries(value).filter(([key]) => { + return key.trim() !== ''; + }) + ); + +/** + * Package names in package-lock v3 are prefixed with `node_modules/`. + * This function removes that prefix to extract only the package name. + */ +const removeNodeModulesPrefix = (packageName: string): string => + packageName.replace(/^node_modules\//, ''); + +export const PackageLockV3Schema = z.object({ + lockfileVersion: z.literal(3), + packages: z + .record( + z.string().transform(removeNodeModulesPrefix), + z.object({ version: z.string() }) + ) + .transform(removeRecordsWithEmptyKeys), +}); + +export const PackageLockPreV3Schema = z.object({ + lockfileVersion: z.union([z.literal(2), z.literal(1)]), + dependencies: z + .record(z.string(), z.object({ version: z.string() })) + .optional(), +}); From 5bb44aeb4e0b878eb96df24a1269729cfc98ca82 Mon Sep 17 00:00:00 2001 From: Matthias Junker Date: Mon, 22 May 2023 14:04:34 +0200 Subject: [PATCH 05/14] chore: review findings --- lib/modules/manager/npm/extract/locked-versions.ts | 11 +++++------ lib/modules/manager/npm/extract/npm.spec.ts | 12 ++++++------ 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/lib/modules/manager/npm/extract/locked-versions.ts b/lib/modules/manager/npm/extract/locked-versions.ts index 7246c8b1801195..365df961b9ae0e 100644 --- a/lib/modules/manager/npm/extract/locked-versions.ts +++ b/lib/modules/manager/npm/extract/locked-versions.ts @@ -80,14 +80,13 @@ export async function getLockedVersions( } else { packageFile.extractedConstraints!.npm = '<9'; } - } else if ( - lockfileVersion === 3 && - !packageFile.extractedConstraints?.npm - ) { - packageFile.extractedConstraints!.npm = '>=7'; + } else if (lockfileVersion === 3) { + if (!packageFile.extractedConstraints?.npm) { + packageFile.extractedConstraints!.npm = '>=7'; + } } else { logger.warn( - { lockfileVersion }, + { lockfileVersion, npmLock }, 'Found unsupported npm lockfile version' ); return; diff --git a/lib/modules/manager/npm/extract/npm.spec.ts b/lib/modules/manager/npm/extract/npm.spec.ts index 238badaa780cab..55f35efc9992ae 100644 --- a/lib/modules/manager/npm/extract/npm.spec.ts +++ b/lib/modules/manager/npm/extract/npm.spec.ts @@ -88,16 +88,16 @@ describe('modules/manager/npm/extract/npm', () => { }); it('should not throw if additional property exists', () => { - const npm9Lock = Fixtures.get('npm9/package-lock.json', '..'); + const npm9Lock = Fixtures.getJson('npm9/package-lock.json', '..'); expect(() => - extractPackages({ ...JSON.parse(npm9Lock), additionalProperty: 'test' }) + extractPackages({ ...npm9Lock, additionalProperty: 'test' }) ).not.toThrow(); }); it('should throw if lockfileVersion is invalid', () => { - const npm9Lock = Fixtures.get('npm9/package-lock.json', '..'); + const npm9Lock = Fixtures.getJson('npm9/package-lock.json', '..'); expect(() => { - extractPackages({ ...JSON.parse(npm9Lock), lockfileVersion: 4 }); + extractPackages({ ...npm9Lock, lockfileVersion: 4 }); }).toThrow( 'Invalid package-lock file. Neither v1, v2 nor v3 schema matched' ); @@ -112,10 +112,10 @@ describe('modules/manager/npm/extract/npm', () => { }); it('should throw if lockfileVersion 3 without packages property', () => { - const npm9Lock = Fixtures.get('npm9/package-lock.json', '..'); + const npm9Lock = Fixtures.getJson('npm9/package-lock.json', '..'); expect(() => { extractPackages({ - ...JSON.parse(npm9Lock), + ...npm9Lock, packages: undefined, lockfileVersion: 3, }); From c4dd27488fbc98c0fa44a340f0921ec08a229b2e Mon Sep 17 00:00:00 2001 From: Matthias Junker Date: Tue, 23 May 2023 08:20:26 +0200 Subject: [PATCH 06/14] chore: added missing test --- .../npm/extract/locked-versions.spec.ts | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/lib/modules/manager/npm/extract/locked-versions.spec.ts b/lib/modules/manager/npm/extract/locked-versions.spec.ts index 252c3fa94733d5..0fda21181846a4 100644 --- a/lib/modules/manager/npm/extract/locked-versions.spec.ts +++ b/lib/modules/manager/npm/extract/locked-versions.spec.ts @@ -1,3 +1,4 @@ +import { logger } from '../../../../../test/util'; import type { PackageFile } from '../../types'; import type { NpmManagerData } from '../types'; import { getLockedVersions } from './locked-versions'; @@ -536,6 +537,35 @@ describe('modules/manager/npm/extract/locked-versions', () => { ]); }); + it('should log warning if unsupported lockfileVersion is found', async () => { + npm.getNpmLock.mockReturnValue({ + lockedVersions: {}, + lockfileVersion: 99, + }); + const packageFiles = [ + { + managerData: { + npmLock: 'package-lock.json', + }, + extractedConstraints: {}, + deps: [ + { depName: 'a', currentValue: '1.0.0' }, + { depName: 'b', currentValue: '2.0.0' }, + ], + packageFile: 'some-file', + }, + ]; + await getLockedVersions(packageFiles); + expect(packageFiles).toEqual(packageFiles); + expect(logger.logger.warn).toHaveBeenCalledWith( + { + lockfileVersion: 99, + npmLock: 'package-lock.json', + }, + 'Found unsupported npm lockfile version' + ); + }); + describe('lockfileVersion 3', () => { it('uses package-lock.json with npm v9.0.0', async () => { npm.getNpmLock.mockReturnValue({ From 6820a532a64db84c5e42624bf5400219849e93b2 Mon Sep 17 00:00:00 2001 From: Matt Date: Wed, 24 May 2023 12:57:55 +0200 Subject: [PATCH 07/14] Update lib/modules/manager/npm/extract/npm.spec.ts Co-authored-by: Michael Kriese --- lib/modules/manager/npm/extract/npm.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/manager/npm/extract/npm.spec.ts b/lib/modules/manager/npm/extract/npm.spec.ts index 55f35efc9992ae..bac87fe729a66b 100644 --- a/lib/modules/manager/npm/extract/npm.spec.ts +++ b/lib/modules/manager/npm/extract/npm.spec.ts @@ -50,7 +50,7 @@ describe('modules/manager/npm/extract/npm', () => { it('extracts npm 9 lockfile', async () => { const npm9Lock = Fixtures.get('npm9/package-lock.json', '..'); - fs.readLocalFile.mockResolvedValueOnce(npm9Lock as never); + fs.readLocalFile.mockResolvedValueOnce(npm9Lock); const res = await getNpmLock('package.json'); expect(res).toEqual({ lockedVersions: { From a219b7377c0a60238ed8ca4dc758ae7e2f130b30 Mon Sep 17 00:00:00 2001 From: Matt Date: Wed, 24 May 2023 12:58:04 +0200 Subject: [PATCH 08/14] Update lib/modules/manager/npm/extract/npm.ts Co-authored-by: Michael Kriese --- lib/modules/manager/npm/extract/npm.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/manager/npm/extract/npm.ts b/lib/modules/manager/npm/extract/npm.ts index ac9c5086b7d88d..28b54892dcbe9b 100644 --- a/lib/modules/manager/npm/extract/npm.ts +++ b/lib/modules/manager/npm/extract/npm.ts @@ -21,7 +21,7 @@ export async function getNpmLock(filePath: string): Promise { } } -export function extractPackages(lockParsed: any): { +export function extractPackages(lockParsed: unknown): { packages: LockFileEntry; lockfileVersion: number; } { From d9b801ae0fcb3092a790011503f638f8f99a846f Mon Sep 17 00:00:00 2001 From: Matt Date: Wed, 24 May 2023 12:58:55 +0200 Subject: [PATCH 09/14] Update lib/modules/manager/npm/extract/schema.ts Co-authored-by: Sergei Zharinov --- lib/modules/manager/npm/extract/schema.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/manager/npm/extract/schema.ts b/lib/modules/manager/npm/extract/schema.ts index c95261588cf3b0..2e76b04599d15b 100644 --- a/lib/modules/manager/npm/extract/schema.ts +++ b/lib/modules/manager/npm/extract/schema.ts @@ -32,5 +32,5 @@ export const PackageLockPreV3Schema = z.object({ lockfileVersion: z.union([z.literal(2), z.literal(1)]), dependencies: z .record(z.string(), z.object({ version: z.string() })) - .optional(), + .catch({}), }); From 52aee73a271b8ece457a169b8b3c98a12bcd6f80 Mon Sep 17 00:00:00 2001 From: Matt Date: Thu, 25 May 2023 09:16:27 +0200 Subject: [PATCH 10/14] Update lib/modules/manager/npm/extract/schema.ts Co-authored-by: Sergei Zharinov --- lib/modules/manager/npm/extract/schema.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/modules/manager/npm/extract/schema.ts b/lib/modules/manager/npm/extract/schema.ts index 2e76b04599d15b..fc1fd84f6cbf30 100644 --- a/lib/modules/manager/npm/extract/schema.ts +++ b/lib/modules/manager/npm/extract/schema.ts @@ -20,12 +20,13 @@ const removeNodeModulesPrefix = (packageName: string): string => export const PackageLockV3Schema = z.object({ lockfileVersion: z.literal(3), - packages: z - .record( - z.string().transform(removeNodeModulesPrefix), - z.object({ version: z.string() }) - ) - .transform(removeRecordsWithEmptyKeys), + packages: LooseRecord( + z + .string() + .transform((x) => x.replace(/^node_modules\//, '')) + .refine((x) => x.trim() !== ''), + z.object({ version: z.string() }) + ), }); export const PackageLockPreV3Schema = z.object({ From 98f81d4bf0a1447679d9cf15766a1770f3ea474a Mon Sep 17 00:00:00 2001 From: Matthias Junker Date: Thu, 25 May 2023 09:20:39 +0200 Subject: [PATCH 11/14] chore: review findings --- lib/modules/manager/npm/extract/schema.ts | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/lib/modules/manager/npm/extract/schema.ts b/lib/modules/manager/npm/extract/schema.ts index fc1fd84f6cbf30..53a214c5601067 100644 --- a/lib/modules/manager/npm/extract/schema.ts +++ b/lib/modules/manager/npm/extract/schema.ts @@ -1,22 +1,5 @@ import { z } from 'zod'; - -/** - * Removes the entry with an empty key which is used - * in packagelock v3 to indicate a root package. - */ -const removeRecordsWithEmptyKeys = (value: any): any => - Object.fromEntries( - Object.entries(value).filter(([key]) => { - return key.trim() !== ''; - }) - ); - -/** - * Package names in package-lock v3 are prefixed with `node_modules/`. - * This function removes that prefix to extract only the package name. - */ -const removeNodeModulesPrefix = (packageName: string): string => - packageName.replace(/^node_modules\//, ''); +import { LooseRecord } from '../../../../util/schema-utils'; export const PackageLockV3Schema = z.object({ lockfileVersion: z.literal(3), From 476049dd1c83829c4dc7dfeb4522d5b6f61d4a10 Mon Sep 17 00:00:00 2001 From: Matthias Junker Date: Thu, 25 May 2023 09:42:23 +0200 Subject: [PATCH 12/14] chore: adjusted npm to PR 22405 --- lib/modules/manager/npm/extract/locked-versions.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/manager/npm/extract/locked-versions.ts b/lib/modules/manager/npm/extract/locked-versions.ts index 739ae9881b1992..98d73778386175 100644 --- a/lib/modules/manager/npm/extract/locked-versions.ts +++ b/lib/modules/manager/npm/extract/locked-versions.ts @@ -89,7 +89,7 @@ export async function getLockedVersions( } } else if (lockfileVersion === 3) { if (!packageFile.extractedConstraints?.npm) { - packageFile.extractedConstraints!.npm = '>=7'; + npm = '>=7'; } } else { logger.warn( From 5bc565912ea7d55e08c706ad5dcf8f0d0d6f8090 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Tue, 30 May 2023 11:52:09 +0300 Subject: [PATCH 13/14] Refactor schema --- .../manager/npm/extract/locked-versions.ts | 8 ++- lib/modules/manager/npm/extract/npm.spec.ts | 64 +++---------------- lib/modules/manager/npm/extract/npm.ts | 54 +++++----------- lib/modules/manager/npm/extract/schema.ts | 25 ++++++-- 4 files changed, 50 insertions(+), 101 deletions(-) diff --git a/lib/modules/manager/npm/extract/locked-versions.ts b/lib/modules/manager/npm/extract/locked-versions.ts index 6fafd8d5a68d2a..44460ca46c71c6 100644 --- a/lib/modules/manager/npm/extract/locked-versions.ts +++ b/lib/modules/manager/npm/extract/locked-versions.ts @@ -60,7 +60,13 @@ export async function getLockedVersions( lockFiles.push(npmLock); if (!lockFileCache[npmLock]) { logger.trace('Retrieving/parsing ' + npmLock); - lockFileCache[npmLock] = await getNpmLock(npmLock); + const cache = await getNpmLock(npmLock); + // istanbul ignore if + if (!cache) { + logger.warn({ npmLock }, 'Npm: unable to get lockfile'); + return; + } + lockFileCache[npmLock] = cache; } const { lockfileVersion } = lockFileCache[npmLock]; diff --git a/lib/modules/manager/npm/extract/npm.spec.ts b/lib/modules/manager/npm/extract/npm.spec.ts index bac87fe729a66b..c6d4ca3fcd59e3 100644 --- a/lib/modules/manager/npm/extract/npm.spec.ts +++ b/lib/modules/manager/npm/extract/npm.spec.ts @@ -1,15 +1,15 @@ import { Fixtures } from '../../../../../test/fixtures'; import { fs } from '../../../../../test/util'; -import { extractPackages, getNpmLock } from './npm'; +import { getNpmLock } from './npm'; jest.mock('../../../../util/fs'); describe('modules/manager/npm/extract/npm', () => { describe('.getNpmLock()', () => { - it('returns empty if failed to parse', async () => { + it('returns null if failed to parse', async () => { fs.readLocalFile.mockResolvedValueOnce('abcd'); const res = await getNpmLock('package.json'); - expect(Object.keys(res.lockedVersions)).toHaveLength(0); + expect(res).toBeNull(); }); it('extracts', async () => { @@ -66,62 +66,16 @@ describe('modules/manager/npm/extract/npm', () => { }); }); - it('returns empty if no deps', async () => { + it('returns null if no deps', async () => { fs.readLocalFile.mockResolvedValueOnce('{}'); const res = await getNpmLock('package.json'); - expect(Object.keys(res.lockedVersions)).toHaveLength(0); - }); - }); - - describe('.extractPackages', () => { - it('should parse lockfileVersion 1 without dependencies', () => { - expect( - extractPackages({ - name: 'no_dependencies', - version: '1.0.0', - lockfileVersion: 1, - }) - ).toEqual({ - packages: {}, - lockfileVersion: 1, - }); - }); - - it('should not throw if additional property exists', () => { - const npm9Lock = Fixtures.getJson('npm9/package-lock.json', '..'); - expect(() => - extractPackages({ ...npm9Lock, additionalProperty: 'test' }) - ).not.toThrow(); - }); - - it('should throw if lockfileVersion is invalid', () => { - const npm9Lock = Fixtures.getJson('npm9/package-lock.json', '..'); - expect(() => { - extractPackages({ ...npm9Lock, lockfileVersion: 4 }); - }).toThrow( - 'Invalid package-lock file. Neither v1, v2 nor v3 schema matched' - ); + expect(res).toBeNull(); }); - it('should throw if lock file is empty', () => { - expect(() => { - extractPackages({}); - }).toThrow( - 'Invalid package-lock file. Neither v1, v2 nor v3 schema matched' - ); - }); - - it('should throw if lockfileVersion 3 without packages property', () => { - const npm9Lock = Fixtures.getJson('npm9/package-lock.json', '..'); - expect(() => { - extractPackages({ - ...npm9Lock, - packages: undefined, - lockfileVersion: 3, - }); - }).toThrow( - 'Invalid package-lock file. Neither v1, v2 nor v3 schema matched' - ); + it('returns null on read error', async () => { + fs.readLocalFile.mockResolvedValueOnce(null); + const res = await getNpmLock('package.json'); + expect(res).toBeNull(); }); }); }); diff --git a/lib/modules/manager/npm/extract/npm.ts b/lib/modules/manager/npm/extract/npm.ts index 28b54892dcbe9b..0e703a86923123 100644 --- a/lib/modules/manager/npm/extract/npm.ts +++ b/lib/modules/manager/npm/extract/npm.ts @@ -1,47 +1,23 @@ import { logger } from '../../../../logger'; import { readLocalFile } from '../../../../util/fs'; -import { PackageLockPreV3Schema, PackageLockV3Schema } from './schema'; -import type { LockFile, LockFileEntry } from './types'; +import { PackageLock } from './schema'; +import type { LockFile } from './types'; -export async function getNpmLock(filePath: string): Promise { - // TODO #7154 - const lockRaw = (await readLocalFile(filePath, 'utf8'))!; - try { - const lockParsed = JSON.parse(lockRaw); - const lockedVersions: Record = {}; - const { packages, lockfileVersion } = extractPackages(lockParsed); - for (const [entry, val] of Object.entries(packages)) { - logger.trace({ entry, version: val.version }); - lockedVersions[entry] = val.version; - } - return { lockedVersions, lockfileVersion }; - } catch (err) { - logger.debug({ filePath, err }, 'Warning: Exception parsing npm lock file'); - return { lockedVersions: {} }; +export async function getNpmLock(filePath: string): Promise { + const lockfileContent = await readLocalFile(filePath, 'utf8'); + if (!lockfileContent) { + logger.debug({ filePath }, 'Npm: unable to read lockfile'); + return null; } -} - -export function extractPackages(lockParsed: unknown): { - packages: LockFileEntry; - lockfileVersion: number; -} { - const packageLockPreV3ParseResult = - PackageLockPreV3Schema.safeParse(lockParsed); - const packageLockV3ParseResult = PackageLockV3Schema.safeParse(lockParsed); - if (packageLockPreV3ParseResult.success) { - return { - packages: packageLockPreV3ParseResult.data.dependencies ?? {}, - lockfileVersion: packageLockPreV3ParseResult.data.lockfileVersion, - }; - } else if (packageLockV3ParseResult.success) { - return { - packages: packageLockV3ParseResult.data.packages, - lockfileVersion: packageLockV3ParseResult.data.lockfileVersion, - }; - } else { - throw new Error( - 'Invalid package-lock file. Neither v1, v2 nor v3 schema matched' + const parsedLockfile = PackageLock.safeParse(lockfileContent); + if (!parsedLockfile.success) { + logger.debug( + { filePath, err: parsedLockfile.error }, + 'Npm: unable to parse lockfile' ); + return null; } + + return parsedLockfile.data; } diff --git a/lib/modules/manager/npm/extract/schema.ts b/lib/modules/manager/npm/extract/schema.ts index 53a214c5601067..96cde78ca64e19 100644 --- a/lib/modules/manager/npm/extract/schema.ts +++ b/lib/modules/manager/npm/extract/schema.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import { LooseRecord } from '../../../../util/schema-utils'; +import { Json, LooseRecord } from '../../../../util/schema-utils'; export const PackageLockV3Schema = z.object({ lockfileVersion: z.literal(3), @@ -12,9 +12,22 @@ export const PackageLockV3Schema = z.object({ ), }); -export const PackageLockPreV3Schema = z.object({ - lockfileVersion: z.union([z.literal(2), z.literal(1)]), - dependencies: z - .record(z.string(), z.object({ version: z.string() })) - .catch({}), +export const PackageLockPreV3Schema = z + .object({ + lockfileVersion: z.union([z.literal(2), z.literal(1)]), + dependencies: LooseRecord(z.object({ version: z.string() })), + }) + .transform(({ lockfileVersion, dependencies: packages }) => ({ + lockfileVersion, + packages, + })); + +export const PackageLock = Json.pipe( + z.union([PackageLockV3Schema, PackageLockPreV3Schema]) +).transform(({ packages, lockfileVersion }) => { + const lockedVersions: Record = {}; + for (const [entry, val] of Object.entries(packages)) { + lockedVersions[entry] = val.version; + } + return { lockedVersions, lockfileVersion }; }); From 4f12ce9a0d1416c94a4aa17a8699ecc6559b10bf Mon Sep 17 00:00:00 2001 From: Matthias Junker Date: Fri, 2 Jun 2023 09:44:25 +0200 Subject: [PATCH 14/14] chore: fix test after merge issue --- lib/modules/manager/npm/extract/npm.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/modules/manager/npm/extract/npm.ts b/lib/modules/manager/npm/extract/npm.ts index 0e703a86923123..862cabfcbe63ae 100644 --- a/lib/modules/manager/npm/extract/npm.ts +++ b/lib/modules/manager/npm/extract/npm.ts @@ -3,11 +3,11 @@ import { readLocalFile } from '../../../../util/fs'; import { PackageLock } from './schema'; import type { LockFile } from './types'; -export async function getNpmLock(filePath: string): Promise { +export async function getNpmLock(filePath: string): Promise { const lockfileContent = await readLocalFile(filePath, 'utf8'); if (!lockfileContent) { logger.debug({ filePath }, 'Npm: unable to read lockfile'); - return null; + return { lockedVersions: {} }; } const parsedLockfile = PackageLock.safeParse(lockfileContent); @@ -16,7 +16,7 @@ export async function getNpmLock(filePath: string): Promise { { filePath, err: parsedLockfile.error }, 'Npm: unable to parse lockfile' ); - return null; + return { lockedVersions: {} }; } return parsedLockfile.data;