From a3d715981fa593e8faf8f79741a2dc2b354b89ad Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Wed, 3 Jun 2020 12:28:37 +0200 Subject: [PATCH] fix(npm): delete lock files before lock file maintenance --- lib/manager/common.ts | 1 + .../__snapshots__/npm.spec.ts.snap | 22 +++++ .../__snapshots__/pnpm.spec.ts.snap | 22 +++++ .../__snapshots__/yarn.spec.ts.snap | 90 +++++++++++++++++++ lib/manager/npm/post-update/index.ts | 6 +- lib/manager/npm/post-update/npm.spec.ts | 15 ++++ lib/manager/npm/post-update/npm.ts | 17 +++- lib/manager/npm/post-update/pnpm.spec.ts | 11 +++ lib/manager/npm/post-update/pnpm.ts | 27 ++++-- lib/manager/npm/post-update/yarn.spec.ts | 26 ++++++ lib/manager/npm/post-update/yarn.ts | 21 ++++- 11 files changed, 248 insertions(+), 10 deletions(-) diff --git a/lib/manager/common.ts b/lib/manager/common.ts index 7d50db550fef6b..63543e2547fb82 100644 --- a/lib/manager/common.ts +++ b/lib/manager/common.ts @@ -193,6 +193,7 @@ export interface Upgrade> toVersion?: string; updateType?: UpdateType; version?: string; + isLockFileMaintenance?: boolean; } export interface ArtifactError { diff --git a/lib/manager/npm/post-update/__snapshots__/npm.spec.ts.snap b/lib/manager/npm/post-update/__snapshots__/npm.spec.ts.snap index b44fbfc335b49a..0b5f39e7c6cce0 100644 --- a/lib/manager/npm/post-update/__snapshots__/npm.spec.ts.snap +++ b/lib/manager/npm/post-update/__snapshots__/npm.spec.ts.snap @@ -45,6 +45,28 @@ Array [ exports[`generateLockFile performs full install 1`] = `Array []`; +exports[`generateLockFile performs lock file maintenance 1`] = ` +Array [ + Object { + "cmd": "npm install --package-lock-only --ignore-scripts --no-audit", + "options": Object { + "cwd": "some-dir", + "encoding": "utf-8", + "env": Object { + "HOME": "/home/user", + "HTTPS_PROXY": "https://example.com", + "HTTP_PROXY": "http://example.com", + "LANG": "en_US.UTF-8", + "LC_ALL": "en_US", + "NO_PROXY": "localhost", + "PATH": "/tmp/path", + }, + "timeout": 900000, + }, + }, +] +`; + exports[`generateLockFile performs lock file updates 1`] = ` Array [ Object { diff --git a/lib/manager/npm/post-update/__snapshots__/pnpm.spec.ts.snap b/lib/manager/npm/post-update/__snapshots__/pnpm.spec.ts.snap index 35dffb000650ca..06316efe2eb4ac 100644 --- a/lib/manager/npm/post-update/__snapshots__/pnpm.spec.ts.snap +++ b/lib/manager/npm/post-update/__snapshots__/pnpm.spec.ts.snap @@ -65,3 +65,25 @@ Array [ }, ] `; + +exports[`generateLockFile performs lock file maintenance 1`] = ` +Array [ + Object { + "cmd": "pnpm install --lockfile-only --ignore-scripts --ignore-pnpmfile", + "options": Object { + "cwd": "some-dir", + "encoding": "utf-8", + "env": Object { + "HOME": "/home/user", + "HTTPS_PROXY": "https://example.com", + "HTTP_PROXY": "http://example.com", + "LANG": "en_US.UTF-8", + "LC_ALL": "en_US", + "NO_PROXY": "localhost", + "PATH": "/tmp/path", + }, + "timeout": 900000, + }, + }, +] +`; diff --git a/lib/manager/npm/post-update/__snapshots__/yarn.spec.ts.snap b/lib/manager/npm/post-update/__snapshots__/yarn.spec.ts.snap index 350d5818f49f59..3edffc2f3fdb40 100644 --- a/lib/manager/npm/post-update/__snapshots__/yarn.spec.ts.snap +++ b/lib/manager/npm/post-update/__snapshots__/yarn.spec.ts.snap @@ -112,6 +112,96 @@ Array [ ] `; +exports[`manager/npm/post-update/yarn performs lock file maintenance using yarn v1.22.0 1`] = ` +Array [ + Object { + "cmd": "yarn install --ignore-engines --ignore-platform --network-timeout 100000 --ignore-scripts", + "options": Object { + "cwd": "some-dir", + "encoding": "utf-8", + "env": Object { + "HOME": "/home/user", + "HTTPS_PROXY": "https://example.com", + "HTTP_PROXY": "http://example.com", + "LANG": "en_US.UTF-8", + "LC_ALL": "en_US", + "NO_PROXY": "localhost", + "PATH": "/tmp/path", + }, + "timeout": 900000, + }, + }, + Object { + "cmd": "npx yarn-deduplicate --strategy fewer", + "options": Object { + "cwd": "some-dir", + "encoding": "utf-8", + "env": Object { + "HOME": "/home/user", + "HTTPS_PROXY": "https://example.com", + "HTTP_PROXY": "http://example.com", + "LANG": "en_US.UTF-8", + "LC_ALL": "en_US", + "NO_PROXY": "localhost", + "PATH": "/tmp/path", + }, + "timeout": 900000, + }, + }, + Object { + "cmd": "yarn install --ignore-engines --ignore-platform --network-timeout 100000 --ignore-scripts", + "options": Object { + "cwd": "some-dir", + "encoding": "utf-8", + "env": Object { + "HOME": "/home/user", + "HTTPS_PROXY": "https://example.com", + "HTTP_PROXY": "http://example.com", + "LANG": "en_US.UTF-8", + "LC_ALL": "en_US", + "NO_PROXY": "localhost", + "PATH": "/tmp/path", + }, + "timeout": 900000, + }, + }, + Object { + "cmd": "npx yarn-deduplicate --strategy highest", + "options": Object { + "cwd": "some-dir", + "encoding": "utf-8", + "env": Object { + "HOME": "/home/user", + "HTTPS_PROXY": "https://example.com", + "HTTP_PROXY": "http://example.com", + "LANG": "en_US.UTF-8", + "LC_ALL": "en_US", + "NO_PROXY": "localhost", + "PATH": "/tmp/path", + }, + "timeout": 900000, + }, + }, + Object { + "cmd": "yarn install --ignore-engines --ignore-platform --network-timeout 100000 --ignore-scripts", + "options": Object { + "cwd": "some-dir", + "encoding": "utf-8", + "env": Object { + "HOME": "/home/user", + "HTTPS_PROXY": "https://example.com", + "HTTP_PROXY": "http://example.com", + "LANG": "en_US.UTF-8", + "LC_ALL": "en_US", + "NO_PROXY": "localhost", + "PATH": "/tmp/path", + }, + "timeout": 900000, + }, + }, +] +`; + exports[`manager/npm/post-update/yarn performs lock file updates and full install using yarn v1.22.0 1`] = ` Array [ Object { diff --git a/lib/manager/npm/post-update/index.ts b/lib/manager/npm/post-update/index.ts index 842acf60125630..7ba4aae6f2832b 100644 --- a/lib/manager/npm/post-update/index.ts +++ b/lib/manager/npm/post-update/index.ts @@ -572,10 +572,14 @@ export async function getAdditionalFiles( additionalNpmrcContent ); logger.debug(`Generating pnpm-lock.yaml for ${lockFileDir}`); + const upgrades = config.upgrades.filter( + (upgrade) => upgrade.pnpmShrinkwrap === lockFile + ); const res = await pnpm.generateLockFile( upath.join(config.localDir, lockFileDir), env, - config + config, + upgrades ); if (res.error) { // istanbul ignore if diff --git a/lib/manager/npm/post-update/npm.spec.ts b/lib/manager/npm/post-update/npm.spec.ts index ddfb3666f3bba2..eaa0bd421e37b2 100644 --- a/lib/manager/npm/post-update/npm.spec.ts +++ b/lib/manager/npm/post-update/npm.spec.ts @@ -172,4 +172,19 @@ describe('generateLockFile', () => { expect(res.lockFile).toEqual('package-lock-contents'); expect(execSnapshots).toMatchSnapshot(); }); + it('performs lock file maintenance', async () => { + const execSnapshots = mockExecAll(exec); + fs.readFile = jest.fn(() => 'package-lock-contents') as never; + const res = await npmHelper.generateLockFile( + 'some-dir', + {}, + 'package-lock.json', + {}, + [{ isLockFileMaintenance: true }] + ); + expect(fs.readFile).toHaveBeenCalledTimes(1); + expect(fs.remove).toHaveBeenCalledTimes(1); + expect(res.lockFile).toEqual('package-lock-contents'); + expect(execSnapshots).toMatchSnapshot(); + }); }); diff --git a/lib/manager/npm/post-update/npm.ts b/lib/manager/npm/post-update/npm.ts index e597371b140a21..1bfe12b5fc5b2f 100644 --- a/lib/manager/npm/post-update/npm.ts +++ b/lib/manager/npm/post-update/npm.ts @@ -1,4 +1,4 @@ -import { move, pathExists, readFile } from 'fs-extra'; +import { move, pathExists, readFile, remove } from 'fs-extra'; import { validRange } from 'semver'; import { quote } from 'shlex'; import { join } from 'upath'; @@ -88,6 +88,21 @@ export async function generateLockFile( commands.push('npm dedupe'); } + if (upgrades.find((upgrade) => upgrade.isLockFileMaintenance)) { + const lockFileName = join(cwd, filename); + logger.debug( + `Removing ${lockFileName} first due to lock file maintenance upgrade` + ); + try { + await remove(lockFileName); + } catch (err) /* istanbul ignore next */ { + logger.debug( + { err, lockFileName }, + 'Error removing yarn.lock for lock file maintenance' + ); + } + } + // Run the commands await exec(commands, execOptions); diff --git a/lib/manager/npm/post-update/pnpm.spec.ts b/lib/manager/npm/post-update/pnpm.spec.ts index ce28dd90ef0734..8871e4002339c9 100644 --- a/lib/manager/npm/post-update/pnpm.spec.ts +++ b/lib/manager/npm/post-update/pnpm.spec.ts @@ -51,4 +51,15 @@ describe('generateLockFile', () => { expect(res.lockFile).toEqual('package-lock-contents'); expect(execSnapshots).toMatchSnapshot(); }); + it('performs lock file maintenance', async () => { + const execSnapshots = mockExecAll(exec); + fs.readFile = jest.fn(() => 'package-lock-contents') as never; + const res = await pnpmHelper.generateLockFile('some-dir', {}, config, [ + { isLockFileMaintenance: true }, + ]); + expect(fs.readFile).toHaveBeenCalledTimes(1); + expect(fs.remove).toHaveBeenCalledTimes(1); + expect(res.lockFile).toEqual('package-lock-contents'); + expect(execSnapshots).toMatchSnapshot(); + }); }); diff --git a/lib/manager/npm/post-update/pnpm.ts b/lib/manager/npm/post-update/pnpm.ts index c3ea7793071d55..16bab073bbf36a 100644 --- a/lib/manager/npm/post-update/pnpm.ts +++ b/lib/manager/npm/post-update/pnpm.ts @@ -1,10 +1,10 @@ -import { readFile } from 'fs-extra'; +import { readFile, remove } from 'fs-extra'; import { validRange } from 'semver'; import { quote } from 'shlex'; import { join } from 'upath'; import { logger } from '../../../logger'; import { ExecOptions, exec } from '../../../util/exec'; -import { PostUpdateConfig } from '../../common'; +import { PostUpdateConfig, Upgrade } from '../../common'; import { getNodeConstraint } from './node-version'; export interface GenerateLockFileResult { @@ -17,9 +17,11 @@ export interface GenerateLockFileResult { export async function generateLockFile( cwd: string, env: NodeJS.ProcessEnv, - config: PostUpdateConfig + config: PostUpdateConfig, + upgrades: Upgrade[] = [] ): Promise { - logger.debug(`Spawning pnpm install to create ${cwd}/pnpm-lock.yaml`); + const lockFileName = join(cwd, 'pnpm-lock.yaml'); + logger.debug(`Spawning pnpm install to create ${lockFileName}`); let lockFile = null; let stdout: string; let stderr: string; @@ -58,8 +60,23 @@ export async function generateLockFile( args += ' --ignore-pnpmfile'; } logger.debug({ cmd, args }, 'pnpm command'); + + if (upgrades.find((upgrade) => upgrade.isLockFileMaintenance)) { + logger.debug( + `Removing ${lockFileName} first due to lock file maintenance upgrade` + ); + try { + await remove(lockFileName); + } catch (err) /* istanbul ignore next */ { + logger.debug( + { err, lockFileName }, + 'Error removing yarn.lock for lock file maintenance' + ); + } + } + await exec(`${cmd} ${args}`, execOptions); - lockFile = await readFile(join(cwd, 'pnpm-lock.yaml'), 'utf8'); + lockFile = await readFile(lockFileName, 'utf8'); } catch (err) /* istanbul ignore next */ { logger.debug( { diff --git a/lib/manager/npm/post-update/yarn.spec.ts b/lib/manager/npm/post-update/yarn.spec.ts index 5d29e4c1ba04c7..e26f857efd5738 100644 --- a/lib/manager/npm/post-update/yarn.spec.ts +++ b/lib/manager/npm/post-update/yarn.spec.ts @@ -47,6 +47,7 @@ describe(getName(__filename), () => { }; const res = await yarnHelper.generateLockFile('some-dir', {}, config); expect(fs.readFile).toHaveBeenCalledTimes(2); + expect(fs.remove).toHaveBeenCalledTimes(0); expect(res.lockFile).toEqual('package-lock-contents'); expect(fixSnapshots(execSnapshots)).toMatchSnapshot(); } @@ -93,6 +94,31 @@ describe(getName(__filename), () => { expect(fixSnapshots(execSnapshots)).toMatchSnapshot(); } ); + it.each([['1.22.0']])( + 'performs lock file maintenance using yarn v%s', + async (yarnVersion) => { + const execSnapshots = mockExecAll(exec, { + stdout: yarnVersion, + stderr: '', + }); + fs.readFile.mockResolvedValue(null); // .yarnrc + fs.readFile.mockResolvedValue('package-lock-contents' as never); + const config = { + dockerMapDotfiles: true, + compatibility: { + yarn: '^1.10.0', + }, + postUpdateOptions: ['yarnDedupeFewer', 'yarnDedupeHighest'], + }; + const res = await yarnHelper.generateLockFile('some-dir', {}, config, [ + { isLockFileMaintenance: true }, + ]); + expect(fs.readFile).toHaveBeenCalledTimes(2); + expect(fs.remove).toHaveBeenCalledTimes(1); + expect(res.lockFile).toEqual('package-lock-contents'); + expect(fixSnapshots(execSnapshots)).toMatchSnapshot(); + } + ); it('catches errors', async () => { const execSnapshots = mockExecAll(exec, { stdout: '1.9.4', diff --git a/lib/manager/npm/post-update/yarn.ts b/lib/manager/npm/post-update/yarn.ts index 4b1939c836f930..81d2d6e8d7e32a 100644 --- a/lib/manager/npm/post-update/yarn.ts +++ b/lib/manager/npm/post-update/yarn.ts @@ -1,5 +1,5 @@ import is from '@sindresorhus/is'; -import { readFile } from 'fs-extra'; +import { readFile, remove } from 'fs-extra'; import { validRange } from 'semver'; import { quote } from 'shlex'; import { join } from 'upath'; @@ -42,7 +42,8 @@ export async function generateLockFile( config: PostUpdateConfig = {}, upgrades: Upgrade[] = [] ): Promise { - logger.debug(`Spawning yarn install to create ${cwd}/yarn.lock`); + const lockFileName = join(cwd, 'yarn.lock'); + logger.debug(`Spawning yarn install to create ${lockFileName}`); let lockFile = null; try { let installYarn = 'npm i -g yarn'; @@ -114,11 +115,25 @@ export async function generateLockFile( commands.push(`yarn install ${cmdOptions}`.trim()); } + if (upgrades.find((upgrade) => upgrade.isLockFileMaintenance)) { + logger.debug( + `Removing ${lockFileName} first due to lock file maintenance upgrade` + ); + try { + await remove(lockFileName); + } catch (err) /* istanbul ignore next */ { + logger.debug( + { err, lockFileName }, + 'Error removing yarn.lock for lock file maintenance' + ); + } + } + // Run the commands await exec(commands, execOptions); // Read the result - lockFile = await readFile(join(cwd, 'yarn.lock'), 'utf8'); + lockFile = await readFile(lockFileName, 'utf8'); } catch (err) /* istanbul ignore next */ { logger.debug( {