diff --git a/lib/config/migration.spec.ts b/lib/config/migration.spec.ts index 86889d5d6011ff..baa90a4aa50dbc 100644 --- a/lib/config/migration.spec.ts +++ b/lib/config/migration.spec.ts @@ -668,6 +668,38 @@ describe('config/migration', () => { expect(migratedConfig).toMatchSnapshot(); }); + it('it migrates pip-compile', () => { + const config: RenovateConfig = { + 'pip-compile': { + enabled: true, + fileMatch: [ + '(^|/)requirements\\.in$', + '(^|/)requirements-fmt\\.in$', + '(^|/)requirements-lint\\.in$', + '.github/workflows/requirements.in', + '(^|/)debian_packages/private/third_party/requirements\\.in$', + '(^|/).*?requirements.*?\\.in$', + ], + }, + }; + const { isMigrated, migratedConfig } = + configMigration.migrateConfig(config); + expect(isMigrated).toBeTrue(); + expect(migratedConfig).toEqual({ + 'pip-compile': { + enabled: true, + fileMatch: [ + '(^|/)requirements\\.txt$', + '(^|/)requirements-fmt\\.txt$', + '(^|/)requirements-lint\\.txt$', + '.github/workflows/requirements.txt', + '(^|/)debian_packages/private/third_party/requirements\\.txt$', + '(^|/).*?requirements.*?\\.txt$', + ], + }, + }); + }); + it('it migrates gradle-lite', () => { const config: RenovateConfig = { 'gradle-lite': { diff --git a/lib/config/migration.ts b/lib/config/migration.ts index 84822fc8633ff2..67f7d7a9035233 100644 --- a/lib/config/migration.ts +++ b/lib/config/migration.ts @@ -150,6 +150,20 @@ export function migrateConfig(config: RenovateConfig): MigratedConfig { } } } + if ( + is.nonEmptyObject(migratedConfig['pip-compile']) && + is.nonEmptyArray(migratedConfig['pip-compile'].fileMatch) + ) { + migratedConfig['pip-compile'].fileMatch = migratedConfig[ + 'pip-compile' + ].fileMatch.map((fileMatch) => { + const match = fileMatch as string; + if (match.endsWith('.in')) { + return match.replace(/\.in$/, '.txt'); + } + return match.replace(/\.in\$$/, '.txt$'); + }); + } if (is.nonEmptyArray(migratedConfig.matchManagers)) { if (migratedConfig.matchManagers.includes('gradle-lite')) { if (!migratedConfig.matchManagers.includes('gradle')) { diff --git a/lib/modules/manager/pip-compile/__fixtures__/requirementsCustomCommand.txt b/lib/modules/manager/pip-compile/__fixtures__/requirementsCustomCommand.txt new file mode 100644 index 00000000000000..82df200bdde4cd --- /dev/null +++ b/lib/modules/manager/pip-compile/__fixtures__/requirementsCustomCommand.txt @@ -0,0 +1,18 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# ./pip-compile-wrapper requirements.in +# +certifi==2023.11.17 + # via requests +charset-normalizer==3.3.2 + # via requests +idna==3.6 + # via requests +markupsafe==2.1.4 + # via jinja2 +requests==2.31.0 + # via -r requirements.in +urllib3==2.1.0 + # via requests diff --git a/lib/modules/manager/pip-compile/artifacts.spec.ts b/lib/modules/manager/pip-compile/artifacts.spec.ts index 4811afd5afef1d..a22f9aca75dd55 100644 --- a/lib/modules/manager/pip-compile/artifacts.spec.ts +++ b/lib/modules/manager/pip-compile/artifacts.spec.ts @@ -10,7 +10,7 @@ import * as docker from '../../../util/exec/docker'; import type { StatusResult } from '../../../util/git/types'; import * as _datasource from '../../datasource'; import type { UpdateArtifactsConfig } from '../types'; -import { constructPipCompileCmd, extractResolver } from './artifacts'; +import { constructPipCompileCmd } from './artifacts'; import { updateArtifacts } from '.'; const datasource = mocked(_datasource); @@ -22,6 +22,18 @@ jest.mock('../../../util/host-rules', () => mockDeep()); jest.mock('../../../util/http'); jest.mock('../../datasource', () => mockDeep()); +function getCommandInHeader(command: string) { + return `# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# ${command} +# +`; +} + +const simpleHeader = getCommandInHeader('pip-compile requirements.in'); + const adminConfig: RepoGlobalConfig = { // `join` fixes Windows CI localDir: join('/tmp/github/some/repo'), @@ -57,48 +69,77 @@ describe('modules/manager/pip-compile/artifacts', () => { packageFileName: 'requirements.in', updatedDeps: [], newPackageFileContent: '', - config, + config: { + ...config, + lockFiles: ['requirements.txt'], + }, }), ).toBeNull(); expect(execSnapshots).toEqual([]); }); it('returns null if unchanged', async () => { - fs.readLocalFile.mockResolvedValueOnce('content'); + fs.readLocalFile.mockResolvedValueOnce(simpleHeader); const execSnapshots = mockExecAll(); - fs.readLocalFile.mockResolvedValueOnce('content'); + fs.readLocalFile.mockResolvedValueOnce('new lock'); expect( await updateArtifacts({ packageFileName: 'requirements.in', updatedDeps: [], newPackageFileContent: 'some new content', - config, + config: { + ...config, + lockFiles: ['requirements.txt'], + }, }), ).toBeNull(); expect(execSnapshots).toMatchObject([ - { cmd: 'pip-compile requirements.in' }, + { cmd: 'pip-compile --no-emit-index-url requirements.in' }, ]); }); + it('returns null if no config.lockFiles', async () => { + fs.readLocalFile.mockResolvedValueOnce(simpleHeader); + fs.readLocalFile.mockResolvedValueOnce('new lock'); + expect( + await updateArtifacts({ + packageFileName: 'requirements.in', + updatedDeps: [], + newPackageFileContent: 'some new content', + config: { + ...config, + }, + }), + ).toBeNull(); + expect(logger.warn).toHaveBeenCalledWith( + { packageFileName: 'requirements.in' }, + 'pip-compile: No lock files associated with a package file', + ); + }); + it('returns updated requirements.txt', async () => { - fs.readLocalFile.mockResolvedValueOnce('current requirements.txt'); + fs.readLocalFile.mockResolvedValueOnce(simpleHeader); const execSnapshots = mockExecAll(); git.getRepoStatus.mockResolvedValue( partial({ modified: ['requirements.txt'], }), ); - fs.readLocalFile.mockResolvedValueOnce('New requirements.txt'); + fs.readLocalFile.mockResolvedValueOnce('new lock'); expect( await updateArtifacts({ packageFileName: 'requirements.in', updatedDeps: [], newPackageFileContent: 'some new content', - config: { ...config, constraints: { python: '3.7' } }, + config: { + ...config, + constraints: { python: '3.7' }, + lockFiles: ['requirements.txt'], + }, }), ).not.toBeNull(); expect(execSnapshots).toMatchObject([ - { cmd: 'pip-compile requirements.in' }, + { cmd: 'pip-compile --no-emit-index-url requirements.in' }, ]); }); @@ -114,14 +155,18 @@ describe('modules/manager/pip-compile/artifacts', () => { modified: ['requirements.txt'], }), ); - fs.readLocalFile.mockResolvedValueOnce('new lock'); + fs.readLocalFile.mockResolvedValueOnce(simpleHeader); fs.ensureCacheDir.mockResolvedValueOnce('/tmp/renovate/cache/others/pip'); expect( await updateArtifacts({ packageFileName: 'requirements.in', updatedDeps: [], newPackageFileContent: 'some new content', - config: { ...config, constraints: { python: '3.10.2' } }, + config: { + ...config, + constraints: { python: '3.10.2' }, + lockFiles: ['requirements.txt'], + }, }), ).not.toBeNull(); @@ -142,7 +187,7 @@ describe('modules/manager/pip-compile/artifacts', () => { '&& ' + 'install-tool pip-tools 6.13.0 ' + '&& ' + - 'pip-compile requirements.in' + + 'pip-compile --no-emit-index-url requirements.in' + '"', }, ]); @@ -160,13 +205,17 @@ describe('modules/manager/pip-compile/artifacts', () => { modified: ['requirements.txt'], }), ); - fs.readLocalFile.mockResolvedValueOnce('new lock'); + fs.readLocalFile.mockResolvedValueOnce(simpleHeader); expect( await updateArtifacts({ packageFileName: 'requirements.in', updatedDeps: [], newPackageFileContent: 'some new content', - config: { ...config, constraints: { python: '3.10.2' } }, + config: { + ...config, + constraints: { python: '3.10.2' }, + lockFiles: ['requirements.txt'], + }, }), ).not.toBeNull(); @@ -174,7 +223,7 @@ describe('modules/manager/pip-compile/artifacts', () => { { cmd: 'install-tool python 3.10.2' }, { cmd: 'install-tool pip-tools 6.13.0' }, { - cmd: 'pip-compile requirements.in', + cmd: 'pip-compile --no-emit-index-url requirements.in', options: { cwd: '/tmp/github/some/repo' }, }, ]); @@ -191,7 +240,7 @@ describe('modules/manager/pip-compile/artifacts', () => { packageFileName: 'requirements.in', updatedDeps: [], newPackageFileContent: '{}', - config, + config: { ...config, lockFiles: ['requirements.txt'] }, }), ).toEqual([ { @@ -202,7 +251,7 @@ describe('modules/manager/pip-compile/artifacts', () => { }); it('returns updated requirements.txt when doing lockfile maintenance', async () => { - fs.readLocalFile.mockResolvedValueOnce('Current requirements.txt'); + fs.readLocalFile.mockResolvedValueOnce(simpleHeader); const execSnapshots = mockExecAll(); git.getRepoStatus.mockResolvedValue( partial({ @@ -215,15 +264,16 @@ describe('modules/manager/pip-compile/artifacts', () => { packageFileName: 'requirements.in', updatedDeps: [], newPackageFileContent: '{}', - config: lockMaintenanceConfig, + config: { ...lockMaintenanceConfig, lockFiles: ['requirements.txt'] }, }), ).not.toBeNull(); expect(execSnapshots).toMatchObject([ - { cmd: 'pip-compile requirements.in' }, + { cmd: 'pip-compile --no-emit-index-url requirements.in' }, ]); }); it('uses pip-compile version from config', async () => { + fs.readLocalFile.mockResolvedValueOnce(simpleHeader); GlobalConfig.set(dockerAdminConfig); // pip-tools datasource.getPkgReleases.mockResolvedValueOnce({ @@ -245,6 +295,7 @@ describe('modules/manager/pip-compile/artifacts', () => { config: { ...config, constraints: { python: '3.10.2', pipTools: '6.13.0' }, + lockFiles: ['requirements.txt'], }, }), ).not.toBeNull(); @@ -266,40 +317,27 @@ describe('modules/manager/pip-compile/artifacts', () => { '&& ' + 'install-tool pip-tools 6.13.0 ' + '&& ' + - 'pip-compile requirements.in' + + 'pip-compile --no-emit-index-url requirements.in' + '"', }, ]); }); describe('constructPipCompileCmd()', () => { - it('returns default cmd for garbage', () => { - expect( + it('throws for garbage', () => { + expect(() => constructPipCompileCmd( Fixtures.get('requirementsNoHeaders.txt'), - 'subdir/requirements.in', 'subdir/requirements.txt', false, ), - ).toBe('pip-compile requirements.in'); - }); - - it('returns --no-emit-index-url when credentials are present in URLs', () => { - expect( - constructPipCompileCmd( - Fixtures.get('requirementsNoHeaders.txt'), - 'subdir/requirements.in', - 'subdir/requirements.txt', - true, - ), - ).toBe('pip-compile --no-emit-index-url requirements.in'); + ).toThrow(/extract/); }); it('returns extracted common arguments (like those featured in the README)', () => { expect( constructPipCompileCmd( Fixtures.get('requirementsWithHashes.txt'), - 'subdir/requirements.in', 'subdir/requirements.txt', false, ), @@ -312,7 +350,6 @@ describe('modules/manager/pip-compile/artifacts', () => { expect( constructPipCompileCmd( Fixtures.get('requirementsWithHashes.txt'), - 'subdir/requirements.in', 'subdir/requirements.txt', true, ), @@ -321,63 +358,55 @@ describe('modules/manager/pip-compile/artifacts', () => { ); }); - it('skips unknown arguments', () => { + it('safeguard against index url leak if not explicitly set by an option', () => { + expect( + constructPipCompileCmd(simpleHeader, 'subdir/requirements.txt', false), + ).toBe('pip-compile --no-emit-index-url requirements.in'); + }); + + it('allow explicit --emit-index-url', () => { expect( constructPipCompileCmd( - Fixtures.get('requirementsWithUnknownArguments.txt'), - 'subdir/requirements.in', + getCommandInHeader('pip-compile --emit-index-url requirements.in'), 'subdir/requirements.txt', false, ), - ).toBe('pip-compile --generate-hashes requirements.in'); - expect(logger.trace).toHaveBeenCalledWith( - { argument: '--version' }, - 'pip-compile argument is not (yet) supported', - ); - expect(logger.warn).toHaveBeenCalledWith( - { argument: '--resolver=foobar' }, - 'pip-compile was previously executed with an unexpected `--resolver` value', - ); + ).toBe('pip-compile --emit-index-url requirements.in'); }); - it('skips exploitable subcommands and files', () => { + // TODO(not7cd): remove when relative pahts are supported + it('change --output-file if differs', () => { expect( constructPipCompileCmd( - Fixtures.get('requirementsWithExploitingArguments.txt'), - 'subdir/requirements.in', + getCommandInHeader( + 'pip-compile --output-file=hey.txt requirements.in', + ), 'subdir/requirements.txt', false, ), ).toBe( - 'pip-compile --generate-hashes --output-file=requirements.txt requirements.in', - ); - expect(logger.warn).toHaveBeenCalledWith( - { argument: '--output-file=/etc/shadow' }, - 'pip-compile was previously executed with an unexpected `--output-file` filename', + 'pip-compile --no-emit-index-url --output-file=requirements.txt requirements.in', ); }); - }); - describe('extractResolver()', () => { - it.each([ - ['--resolver=backtracking', 'backtracking'], - ['--resolver=legacy', 'legacy'], - ])( - 'returns expected value for supported %s resolver', - (argument: string, expected: string) => { - expect(extractResolver(argument)).toBe(expected); - }, - ); + it('throws on unknown arguments', () => { + expect(() => + constructPipCompileCmd( + Fixtures.get('requirementsWithUnknownArguments.txt'), + 'subdir/requirements.txt', + false, + ), + ).toThrow(/supported/); + }); - it.each(['--resolver=foo', '--resolver='])( - 'returns null for unsupported %s resolver', - (argument: string) => { - expect(extractResolver(argument)).toBeNull(); - expect(logger.warn).toHaveBeenCalledWith( - { argument }, - 'pip-compile was previously executed with an unexpected `--resolver` value', - ); - }, - ); + it('throws on custom command', () => { + expect(() => + constructPipCompileCmd( + Fixtures.get('requirementsCustomCommand.txt'), + 'subdir/requirements.txt', + false, + ), + ).toThrow(/custom/); + }); }); }); diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index 31633e75fc2837..ed9a9133b09caa 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -1,4 +1,4 @@ -import { quote, split } from 'shlex'; +import { quote } from 'shlex'; import upath from 'upath'; import { TEMPORARY_ERROR } from '../../../constants/error-messages'; import { logger } from '../../../logger'; @@ -9,61 +9,54 @@ import { writeLocalFile, } from '../../../util/fs'; import { getRepoStatus } from '../../../util/git'; -import { regEx } from '../../../util/regex'; import * as pipRequirements from '../pip_requirements'; import type { UpdateArtifact, UpdateArtifactsResult } from '../types'; import { - constraintLineRegex, - deprecatedAllowedPipArguments, + extractHeaderCommand, getExecOptions, getRegistryUrlVarsFromPackageFile, } from './common'; export function constructPipCompileCmd( content: string, - inputFileName: string, outputFileName: string, haveCredentials: boolean, ): string { - const headers = constraintLineRegex.exec(content); - const args = ['pip-compile']; - if (!!headers?.groups || haveCredentials) { - logger.debug(`Found pip-compile header: ${headers?.[0]}`); - const headerArguments = split(headers?.groups?.arguments ?? ''); - if (haveCredentials && !headerArguments.includes('--no-emit-index-url')) { - headerArguments.push('--no-emit-index-url'); - } - for (const argument of headerArguments) { - if (deprecatedAllowedPipArguments.includes(argument)) { - args.push(argument); - } else if (argument.startsWith('--output-file=')) { - const file = upath.parse(outputFileName).base; - if (argument !== `--output-file=${file}`) { - // we don't trust the user-supplied output-file argument; use our value here - logger.warn( - { argument }, - 'pip-compile was previously executed with an unexpected `--output-file` filename', - ); - } - args.push(`--output-file=${file}`); - } else if (argument.startsWith('--resolver=')) { - const value = extractResolver(argument); - if (value) { - args.push(`--resolver=${value}`); + const headerArguments = extractHeaderCommand(content, outputFileName); + if (headerArguments.isCustomCommand) { + throw new Error( + 'Detected custom command, header modified or set by CUSTOM_COMPILE_COMMAND', + ); + } + if (headerArguments.outputFile) { + // TODO(not7cd): This file path can be relative like `reqs/main.txt` + const file = upath.parse(outputFileName).base; + if (headerArguments.outputFile !== file) { + // we don't trust the user-supplied output-file argument; + // TODO(not7cd): allow relative paths + logger.warn( + { outputFile: headerArguments.outputFile, actualPath: file }, + 'pip-compile was previously executed with an unexpected `--output-file` filename', + ); + // TODO(not7cd): this shouldn't be changed in extract function + headerArguments.outputFile = file; + headerArguments.argv.forEach((item, i) => { + if (item.startsWith('--output-file=')) { + headerArguments.argv[i] = `--output-file=${quote(file)}`; } - } else if (argument.startsWith('--')) { - logger.trace( - { argument }, - 'pip-compile argument is not (yet) supported', - ); - } else { - // ignore position argument (.in file) - } + }); } + } else { + logger.debug(`pip-compile: implicit output file (${outputFileName})`); } - args.push(upath.parse(inputFileName).base); - - return args.map((argument) => quote(argument)).join(' '); + // safeguard against index url leak if not explicitly set by an option + if ( + (!headerArguments.noEmitIndexUrl && !headerArguments.emitIndexUrl) || + (!headerArguments.noEmitIndexUrl && haveCredentials) + ) { + headerArguments.argv.splice(1, 0, '--no-emit-index-url'); + } + return headerArguments.argv.map(quote).join(' '); } export async function updateArtifacts({ @@ -71,10 +64,6 @@ export async function updateArtifacts({ newPackageFileContent: newInputContent, config, }: UpdateArtifact): Promise { - const outputFileName = inputFileName.replace(regEx(/(\.in)?$/), '.txt'); - config.lockFiles = [outputFileName]; - // TODO: remove above and below line and use config.lockFiles directly in the next PR - // istanbul ignore if if (!config.lockFiles) { logger.warn( { packageFileName: inputFileName }, @@ -104,7 +93,6 @@ export async function updateArtifacts({ const registryUrlVars = getRegistryUrlVarsFromPackageFile(packageFile); const cmd = constructPipCompileCmd( existingOutput, - inputFileName, outputFileName, registryUrlVars.haveCredentials, ); @@ -144,16 +132,3 @@ export async function updateArtifacts({ logger.debug('pip-compile: Returning updated output file(s)'); return result; } - -export function extractResolver(argument: string): string | null { - const value = argument.replace('--resolver=', ''); - if (['backtracking', 'legacy'].includes(value)) { - return value; - } - - logger.warn( - { argument }, - 'pip-compile was previously executed with an unexpected `--resolver` value', - ); - return null; -} diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 3279f1473068db..cef57dee2b9c3a 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -7,7 +7,11 @@ import { ensureCacheDir } from '../../../util/fs'; import * as hostRules from '../../../util/host-rules'; import { regEx } from '../../../util/regex'; import type { PackageFileContent, UpdateArtifactsConfig } from '../types'; -import type { GetRegistryUrlVarsResult, PipCompileArgs } from './types'; +import type { + DependencyBetweenFiles, + GetRegistryUrlVarsResult, + PipCompileArgs, +} from './types'; export function getPythonConstraint( config: UpdateArtifactsConfig, @@ -66,13 +70,6 @@ export const constraintLineRegex = regEx( /^(#.*?\r?\n)+# {4}(?\S*)(? .*?)?\r?\n/, ); -// TODO(not7cd): remove in next PR, in favor of extractHeaderCommand -export const deprecatedAllowedPipArguments = [ - '--allow-unsafe', - '--generate-hashes', - '--no-emit-index-url', - '--strip-extras', -]; export const disallowedPipOptions = [ '--no-header', // header is required by this manager ]; @@ -200,7 +197,6 @@ function throwForDisallowedOption(arg: string): void { throw new Error(`Option ${arg} not allowed for this manager`); } } - function throwForNoEqualSignInOptionWithArgument(arg: string): void { if (optionsWithArguments.includes(arg)) { throw new Error( @@ -208,7 +204,6 @@ function throwForNoEqualSignInOptionWithArgument(arg: string): void { ); } } - function throwForUnknownOption(arg: string): void { if (arg.includes('=')) { const [option] = arg.split('='); @@ -222,6 +217,24 @@ function throwForUnknownOption(arg: string): void { throw new Error(`Option ${arg} not supported (yet)`); } +export function generateMermaidGraph( + depsBetweenFiles: DependencyBetweenFiles[], + lockFileArgs: Map, +): string { + const lockFiles = []; + for (const lockFile of lockFileArgs.keys()) { + // TODO: add extra args to the lock file ${extraArgs ? '\n' + extraArgs : ''} + // const extraArgs = pipCompileArgs.extra + // ?.map((v) => '--extra=' + v) + // .join('\n'); + lockFiles.push(` ${lockFile}[[${lockFile}]]`); + } + const edges = depsBetweenFiles.map(({ sourceFile, outputFile, type }) => { + return ` ${sourceFile} -${type === 'constraint' ? '.' : ''}-> ${outputFile}`; + }); + return `graph TD\n${lockFiles.join('\n')}\n${edges.join('\n')}`; +} + function buildRegistryUrl(url: string): URL | null { try { const ret = new URL(url); diff --git a/lib/modules/manager/pip-compile/extract.spec.ts b/lib/modules/manager/pip-compile/extract.spec.ts index c193da74577d40..fa08de855adb6e 100644 --- a/lib/modules/manager/pip-compile/extract.spec.ts +++ b/lib/modules/manager/pip-compile/extract.spec.ts @@ -1,8 +1,20 @@ import { Fixtures } from '../../../../test/fixtures'; -import { extractPackageFile } from '.'; +import { fs } from '../../../../test/util'; +import { extractAllPackageFiles, extractPackageFile } from '.'; jest.mock('../../../util/fs'); +function getSimpleRequirementsFile(command: string, deps: string[] = []) { + return `# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# ${command} +# + +${deps.join('\n')}`; +} + describe('modules/manager/pip-compile/extract', () => { describe('extractPackageFile()', () => { it('returns object for requirements.in', () => { @@ -27,4 +39,102 @@ describe('modules/manager/pip-compile/extract', () => { expect(extractPackageFile('some content', file, {})).toBeNull(); }); }); + + describe('extractAllPackageFiles()', () => { + it('support package file with multiple lock files', async () => { + fs.readLocalFile.mockResolvedValueOnce( + getSimpleRequirementsFile( + 'pip-compile --output-file=requirements1.txt requirements.in', + ['foo==1.0.1'], + ), + ); + // requirements.in is parsed only once + fs.readLocalFile.mockResolvedValueOnce('foo>=1.0.0'); + fs.readLocalFile.mockResolvedValueOnce( + getSimpleRequirementsFile( + 'pip-compile --output-file=requirements2.txt requirements.in', + ['foo==1.0.2'], + ), + ); + fs.readLocalFile.mockResolvedValueOnce( + getSimpleRequirementsFile( + 'pip-compile --output-file=requirements3.txt requirements.in', + ['foo==1.0.3'], + ), + ); + + const lockFiles = [ + 'requirements1.txt', + 'requirements2.txt', + 'requirements3.txt', + ]; + const packageFiles = await extractAllPackageFiles({}, lockFiles); + expect(packageFiles).toBeDefined(); + expect(packageFiles).not.toBeNull(); + expect(packageFiles!.pop()).toHaveProperty('lockFiles', lockFiles); + }); + + it('no lock files in returned package files', async () => { + fs.readLocalFile.mockResolvedValueOnce( + getSimpleRequirementsFile('pip-compile --output-file=foo.txt foo.in', [ + 'foo==1.0.1', + ]), + ); + fs.readLocalFile.mockResolvedValueOnce('foo>=1.0.0'); + fs.readLocalFile.mockResolvedValueOnce( + getSimpleRequirementsFile( + 'pip-compile --output-file=bar.txt bar.in foo.txt', + ['foo==1.0.1', 'bar==2.0.0'], + ), + ); + fs.readLocalFile.mockResolvedValueOnce('bar>=1.0.0'); + + const lockFiles = ['foo.txt', 'bar.txt']; + const packageFiles = await extractAllPackageFiles({}, lockFiles); + expect(packageFiles).toBeDefined(); + packageFiles!.forEach((packageFile) => { + expect(packageFile).not.toHaveProperty('packageFile', 'foo.txt'); + }); + }); + + // TODO(not7cd): update when constraints are supported + it('no constraint files in returned package files', async () => { + fs.readLocalFile.mockResolvedValueOnce( + getSimpleRequirementsFile( + 'pip-compile --output-file=requirements.txt --constraint=constraints.txt requirements.in', + ['foo==1.0.1'], + ), + ); + fs.readLocalFile.mockResolvedValueOnce('foo>=1.0.0'); + + const lockFiles = ['requirements.txt']; + const packageFiles = await extractAllPackageFiles({}, lockFiles); + expect(packageFiles).toBeDefined(); + packageFiles!.forEach((packageFile) => { + expect(packageFile).not.toHaveProperty( + 'packageFile', + 'constraints.txt', + ); + }); + }); + }); + + it('return null for malformed files', async () => { + fs.readLocalFile.mockResolvedValueOnce(''); + fs.readLocalFile.mockResolvedValueOnce( + Fixtures.get('requirementsNoHeaders.txt'), + ); + fs.readLocalFile.mockResolvedValueOnce( + getSimpleRequirementsFile( + 'pip-compile --output-file=foo.txt malformed.in empty.in', + ['foo==1.0.1'], + ), + ); + fs.readLocalFile.mockResolvedValueOnce('!@#$'); + fs.readLocalFile.mockResolvedValueOnce(''); + + const lockFiles = ['empty.txt', 'noHeader.txt', 'badSource.txt']; + const packageFiles = await extractAllPackageFiles({}, lockFiles); + expect(packageFiles).toBeNull(); + }); }); diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index 59c4e13340ca0f..882312499453ec 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -1,10 +1,16 @@ import { logger } from '../../../logger'; +import { readLocalFile } from '../../../util/fs'; import { extractPackageFile as extractRequirementsFile } from '../pip_requirements/extract'; // TODO(not7cd): enable in the next PR, when this can be properly tested // import { extractPackageFile as extractSetupPyFile } from '../pip_setup'; // import { extractPackageFile as extractSetupCfgFile } from '../setup-cfg'; -import type { ExtractConfig, PackageFileContent } from '../types'; -import type { SupportedManagers } from './types'; +import type { ExtractConfig, PackageFile, PackageFileContent } from '../types'; +import { extractHeaderCommand, generateMermaidGraph } from './common'; +import type { + DependencyBetweenFiles, + PipCompileArgs, + SupportedManagers, +} from './types'; function matchManager(filename: string): SupportedManagers | 'unknown' { if (filename.endsWith('setup.py')) { @@ -42,9 +48,9 @@ export function extractPackageFile( case 'unknown': logger.warn( { packageFile }, - `pip-compile: could not determine manager for source file, fallback to pip_requirements`, + `pip-compile: could not determine manager for source file`, ); - return extractRequirementsFile(content); + return null; default: logger.warn( { packageFile, manager }, @@ -53,3 +59,97 @@ export function extractPackageFile( return null; } } + +export async function extractAllPackageFiles( + config: ExtractConfig, + fileMatches: string[], +): Promise { + logger.trace('pip-compile.extractAllPackageFiles()'); + const lockFileArgs = new Map(); + const depsBetweenFiles: DependencyBetweenFiles[] = []; + // for debugging only ^^^ (for now) + const packageFiles = new Map(); + for (const fileMatch of fileMatches) { + const fileContent = await readLocalFile(fileMatch, 'utf8'); + if (!fileContent) { + logger.debug(`pip-compile: no content found for fileMatch ${fileMatch}`); + continue; + } + let pipCompileArgs: PipCompileArgs; + try { + pipCompileArgs = extractHeaderCommand(fileContent, fileMatch); + lockFileArgs.set(fileMatch, pipCompileArgs); + } catch (error) { + logger.warn( + { fileMatch, error }, + 'pip-compile: Failed to extract and parse command in output file header', + ); + continue; + } + for (const constraint in pipCompileArgs.constraintsFiles) { + // TODO(not7cd): handle constraints + /* istanbul ignore next */ + depsBetweenFiles.push({ + sourceFile: constraint, + outputFile: fileMatch, + type: 'constraint', + }); + } + // TODO(not7cd): handle locked deps + // const lockedDeps = extractRequirementsFile(content); + for (const packageFile of pipCompileArgs.sourceFiles) { + depsBetweenFiles.push({ + sourceFile: packageFile, + outputFile: fileMatch, + type: 'requirement', + }); + if (fileMatches.includes(packageFile)) { + // TODO(not7cd): do something about it + logger.warn( + { sourceFile: packageFile, lockFile: fileMatch }, + 'pip-compile: lock file acts as source file for another lock file', + ); + continue; + } + if (packageFiles.has(packageFile)) { + logger.debug( + `pip-compile: ${packageFile} used in multiple output files`, + ); + packageFiles.get(packageFile)!.lockFiles!.push(fileMatch); + continue; + } + const content = await readLocalFile(packageFile, 'utf8'); + if (!content) { + logger.debug(`pip-compile: No content for source file ${packageFile}`); + continue; + } + + const packageFileContent = extractPackageFile( + content, + packageFile, + config, + ); + if (packageFileContent) { + packageFiles.set(packageFile, { + ...packageFileContent, + lockFiles: [fileMatch], + packageFile, + }); + } else { + logger.warn( + { packageFile }, + 'pip-compile: failed to find dependencies in source file', + ); + } + } + } + // TODO(not7cd): sort by requirement layering (-r -c within .in files) + if (packageFiles.size === 0) { + return null; + } + logger.debug( + 'pip-compile: dependency graph:\n' + + generateMermaidGraph(depsBetweenFiles, lockFileArgs), + ); + return Array.from(packageFiles.values()); +} diff --git a/lib/modules/manager/pip-compile/index.ts b/lib/modules/manager/pip-compile/index.ts index 9fb9e6f535491e..b6bc47fca61d1e 100644 --- a/lib/modules/manager/pip-compile/index.ts +++ b/lib/modules/manager/pip-compile/index.ts @@ -2,7 +2,7 @@ import type { Category } from '../../../constants'; import { GitTagsDatasource } from '../../datasource/git-tags'; import { PypiDatasource } from '../../datasource/pypi'; -export { extractPackageFile } from './extract'; +export { extractAllPackageFiles, extractPackageFile } from './extract'; export { updateArtifacts } from './artifacts'; export const supportsLockFileMaintenance = true; diff --git a/lib/modules/manager/pip-compile/readme.md b/lib/modules/manager/pip-compile/readme.md index 09589f039f9ab7..9a3f2ee393753c 100644 --- a/lib/modules/manager/pip-compile/readme.md +++ b/lib/modules/manager/pip-compile/readme.md @@ -12,20 +12,42 @@ You can "activate" the manager by specifying a `fileMatch` pattern such as: ```json { "pip-compile": { - "fileMatch": ["(^|/)requirements\\.in$"] + "fileMatch": ["(^|/)requirements\\.txt$"] } } ``` -### Assumption of `.in`/`.txt` +### Assumption of header with a command -If Renovate matches/extracts a file, it assumes that the corresponding output file is found by swapping the `.in` for `.txt`. -e.g. `requirements.in` => `requirements.txt` -It will not work if files are in separate directories, including `input/requirements.in` and `output/requirements.txt`. +If Renovate matches a `pip-compile` output file it will extract original command that was used to create it from header in this file. Because of that `pip-compile` manager poses restrictions on how this file is generated: -If no `.in` suffix is found, then a `.txt` suffix is appended for the output file, e.g. `foo.file` would look for a corresponding `foo.file.txt`. +- Use default header generation, don't use `--no-header` option. +- Pass all source files explicitly. -We intend to make the mapping configurable in future iterations. +In turn `pip-compile` manager will find all source files and parse them as package files. Currently only `*.in` files associated with `pip_requirements` manager are handled. + +Example header: + +``` +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --no-emit-index-url --output-file=requirements.txt requirements.in +# +``` + +### Conflicts with other managers + +Because `pip-compile` will update source files with their associated manager you should disable them to avoid running these managers twice. + +```json +{ + "pip_requirements": { + "enabled": false + } +} +``` ### Configuration of Python version @@ -44,8 +66,7 @@ To get Renovate to use another version of Python, add a constraints` rule to the Renovate reads the `requirements.txt` file and extracts these `pip-compile` arguments: -- `--generate-hashes` -- `--allow-unsafe` -- `--no-emit-index-url` -- `--strip-extras` -- `--resolver` +- source files as positional arguments +- `--output-file` + +All other allowed `pip-compile` arguments will be passed over without modification. diff --git a/lib/modules/manager/pip-compile/types.ts b/lib/modules/manager/pip-compile/types.ts index 964c1cc9b686ad..91dfcf3a6a59e9 100644 --- a/lib/modules/manager/pip-compile/types.ts +++ b/lib/modules/manager/pip-compile/types.ts @@ -26,3 +26,9 @@ export interface PipCompileArgs { outputFile?: string; sourceFiles: string[]; // positional arguments } + +export interface DependencyBetweenFiles { + sourceFile: string; + outputFile: string; + type: 'requirement' | 'constraint'; +}