From 3362e5147a6c53386c0df3cf4dd9335062e65d82 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Thu, 25 Jan 2024 14:43:06 +0100 Subject: [PATCH 01/85] Change fileMatch behaviour to target pip-compile output files Now package files are infered from command embeded in output file header. This should enable support for additional package managers that use files like setup.py, setup.cfg and those conforming to PEP 621. Command extraction from header has been moved to common module, as it will be reused in lockedDependencyUpdate and other functions. --- .../manager/pip-compile/artifacts.spec.ts | 4 - lib/modules/manager/pip-compile/artifacts.ts | 113 ++----------- lib/modules/manager/pip-compile/common.ts | 155 ++++++++++++++++++ lib/modules/manager/pip-compile/extract.ts | 65 ++++++++ lib/modules/manager/pip-compile/index.ts | 2 +- lib/modules/manager/pip-compile/readme.md | 37 +++-- 6 files changed, 256 insertions(+), 120 deletions(-) create mode 100644 lib/modules/manager/pip-compile/common.ts create mode 100644 lib/modules/manager/pip-compile/extract.ts diff --git a/lib/modules/manager/pip-compile/artifacts.spec.ts b/lib/modules/manager/pip-compile/artifacts.spec.ts index 14a92a54a443e3..fbb09816a415aa 100644 --- a/lib/modules/manager/pip-compile/artifacts.spec.ts +++ b/lib/modules/manager/pip-compile/artifacts.spec.ts @@ -277,7 +277,6 @@ describe('modules/manager/pip-compile/artifacts', () => { expect( constructPipCompileCmd( Fixtures.get('requirementsNoHeaders.txt'), - 'subdir/requirements.in', 'subdir/requirements.txt', ), ).toBe('pip-compile requirements.in'); @@ -287,7 +286,6 @@ describe('modules/manager/pip-compile/artifacts', () => { expect( constructPipCompileCmd( Fixtures.get('requirementsWithHashes.txt'), - 'subdir/requirements.in', 'subdir/requirements.txt', ), ).toBe( @@ -299,7 +297,6 @@ describe('modules/manager/pip-compile/artifacts', () => { expect( constructPipCompileCmd( Fixtures.get('requirementsWithUnknownArguments.txt'), - 'subdir/requirements.in', 'subdir/requirements.txt', ), ).toBe('pip-compile --generate-hashes requirements.in'); @@ -317,7 +314,6 @@ describe('modules/manager/pip-compile/artifacts', () => { expect( constructPipCompileCmd( Fixtures.get('requirementsWithExploitingArguments.txt'), - 'subdir/requirements.in', 'subdir/requirements.txt', ), ).toBe( diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index 15b51aa9fc2cd1..86415f4ceff716 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -1,100 +1,24 @@ -import is from '@sindresorhus/is'; -import { quote, split } from 'shlex'; -import upath from 'upath'; import { TEMPORARY_ERROR } from '../../../constants/error-messages'; import { logger } from '../../../logger'; import { exec } from '../../../util/exec'; import type { ExecOptions } from '../../../util/exec/types'; import { deleteLocalFile, - ensureCacheDir, readLocalFile, writeLocalFile, } from '../../../util/fs'; import { getRepoStatus } from '../../../util/git'; import { regEx } from '../../../util/regex'; -import type { - UpdateArtifact, - UpdateArtifactsConfig, - UpdateArtifactsResult, -} from '../types'; - -function getPythonConstraint( - config: UpdateArtifactsConfig, -): string | undefined | null { - const { constraints = {} } = config; - const { python } = constraints; - - if (python) { - logger.debug('Using python constraint from config'); - return python; - } - - return undefined; -} - -function getPipToolsConstraint(config: UpdateArtifactsConfig): string { - const { constraints = {} } = config; - const { pipTools } = constraints; - - if (is.string(pipTools)) { - logger.debug('Using pipTools constraint from config'); - return pipTools; - } - - return ''; -} - -const constraintLineRegex = regEx( - /^(#.*?\r?\n)+# {4}pip-compile(?.*?)\r?\n/, -); -const allowedPipArguments = [ - '--allow-unsafe', - '--generate-hashes', - '--no-emit-index-url', - '--strip-extras', -]; +import type { UpdateArtifact, UpdateArtifactsResult } from '../types'; +import { extractHeaderCommand, getExecOptions } from './common'; export function constructPipCompileCmd( content: string, - inputFileName: string, outputFileName: string, ): string { - const headers = constraintLineRegex.exec(content); - const args = ['pip-compile']; - if (headers?.groups) { - logger.debug(`Found pip-compile header: ${headers[0]}`); - for (const argument of split(headers.groups.arguments)) { - if (allowedPipArguments.includes(argument)) { - args.push(argument); - } else if (argument.startsWith('--output-file=')) { - const file = upath.parse(outputFileName).base; - if (argument !== `--output-file=${file}`) { - // we don't trust the user-supplied output-file argument; use our value here - logger.warn( - { argument }, - 'pip-compile was previously executed with an unexpected `--output-file` filename', - ); - } - args.push(`--output-file=${file}`); - } else if (argument.startsWith('--resolver=')) { - const value = extractResolver(argument); - if (value) { - args.push(`--resolver=${value}`); - } - } else if (argument.startsWith('--')) { - logger.trace( - { argument }, - 'pip-compile argument is not (yet) supported', - ); - } else { - // ignore position argument (.in file) - } - } - } - args.push(upath.parse(inputFileName).base); - - return args.map((argument) => quote(argument)).join(' '); + const pipCompileArgs = extractHeaderCommand(content, outputFileName); + // TODO(not7cd): sanitize args that require quotes, .map((argument) => quote(argument)) + return pipCompileArgs.argv.join(' '); } export async function updateArtifacts({ @@ -113,33 +37,15 @@ export async function updateArtifacts({ } try { await writeLocalFile(inputFileName, newInputContent); + // TODO(not7cd): check --rebuild and --upgrade option if (config.isLockFileMaintenance) { await deleteLocalFile(outputFileName); } - const cmd = constructPipCompileCmd( - existingOutput, + const cmd = constructPipCompileCmd(existingOutput, outputFileName); + const execOptions: ExecOptions = await getExecOptions( + config, inputFileName, - outputFileName, ); - const constraint = getPythonConstraint(config); - const pipToolsConstraint = getPipToolsConstraint(config); - const execOptions: ExecOptions = { - cwdFile: inputFileName, - docker: {}, - toolConstraints: [ - { - toolName: 'python', - constraint, - }, - { - toolName: 'pip-tools', - constraint: pipToolsConstraint, - }, - ], - extraEnv: { - PIP_CACHE_DIR: await ensureCacheDir('pip'), - }, - }; logger.trace({ cmd }, 'pip-compile command'); await exec(cmd, execOptions); const status = await getRepoStatus(); @@ -173,6 +79,7 @@ export async function updateArtifacts({ } } +// TODO(not7cd): remove, legacy resolver is deprecated and will be removed export function extractResolver(argument: string): string | null { const value = argument.replace('--resolver=', ''); if (['backtracking', 'legacy'].includes(value)) { diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts new file mode 100644 index 00000000000000..8bd749f7da9e1b --- /dev/null +++ b/lib/modules/manager/pip-compile/common.ts @@ -0,0 +1,155 @@ +import is from '@sindresorhus/is'; +import { Command } from 'commander'; +import { split } from 'shlex'; +import upath from 'upath'; +import { logger } from '../../../logger'; +import type { ExecOptions } from '../../../util/exec/types'; +import { ensureCacheDir } from '../../../util/fs'; +import { regEx } from '../../../util/regex'; +import type { UpdateArtifactsConfig } from '../types'; + +export function getPipToolsConstraint(config: UpdateArtifactsConfig): string { + const { constraints = {} } = config; + const { pipTools } = constraints; + + if (is.string(pipTools)) { + logger.debug('Using pipTools constraint from config'); + return pipTools; + } + + return ''; +} +export function getPythonConstraint( + config: UpdateArtifactsConfig, +): string | undefined | null { + const { constraints = {} } = config; + const { python } = constraints; + + if (python) { + logger.debug('Using python constraint from config'); + return python; + } + + return undefined; +} +export async function getExecOptions( + config: UpdateArtifactsConfig, + inputFileName: string, +): Promise { + const constraint = getPythonConstraint(config); + const pipToolsConstraint = getPipToolsConstraint(config); + const execOptions: ExecOptions = { + cwdFile: inputFileName, + docker: {}, + toolConstraints: [ + { + toolName: 'python', + constraint, + }, + { + toolName: 'pip-tools', + constraint: pipToolsConstraint, + }, + ], + extraEnv: { + PIP_CACHE_DIR: await ensureCacheDir('pip'), + }, + }; + return execOptions; +} // TODO(not7cd): rename to getPipToolsVersionConstraint, as constraints have their meaning in pipexport function extractHeaderCommand(content: string): string { + +export const constraintLineRegex = regEx( + /^(#.*?\r?\n)+# {4}(?\S*)(? .*?)?\r?\n/, +); +export const allowedPipArguments = [ + '--allow-unsafe', + '--generate-hashes', + '--no-emit-index-url', // handle this!!! + '--strip-extras', +]; + +// as commander.js is already used, we will reuse it's argument parsing capability +const dummyPipCompile = new Command() + .option('-o, --output-file ') + // .option('--no-emit-index-url') + .option('--extra-index-url') + // .enablePositionalOptions() + .allowUnknownOption() + .allowExcessArguments(); + +interface PipCompileArgs { + command: string; + outputFile?: string; + extra?: string[]; + constraint?: string[]; + sourceFiles: string[]; // positional arguments + argv: string[]; // all arguments as a list +} + +// TODO(not7cd): test on all correct headers, even with CUSTOM_COMPILE_COMMAND +export function extractHeaderCommand( + content: string, + outputFileName: string, +): PipCompileArgs { + const compileCommand = constraintLineRegex.exec(content); + if (compileCommand?.groups) { + logger.debug(`Found pip-compile header: ${compileCommand[0]}`); + } else { + logger.error('Failed to extract command from header'); + // TODO(not7cd): throw + } + const pipCompileArgs: PipCompileArgs = { + argv: [], + command: '', + sourceFiles: [], + }; + if (compileCommand?.groups) { + pipCompileArgs.argv = [compileCommand.groups.command]; + // all arguments are optional, TODO(not7cd): decide if require explicit args + if (compileCommand.groups.arguments) { + pipCompileArgs.argv.push(...split(compileCommand.groups.arguments)); + } + try { + const isCustomCommand = pipCompileArgs.argv[0] !== 'pip-compile'; + const parsedCommand = dummyPipCompile.parse(pipCompileArgs.argv); + const options = parsedCommand.opts(); + // TODO(not7cd): trace unsupported options + const args = parsedCommand.args; + logger.debug( + { + argv: pipCompileArgs.argv, + options, + sourceFiles: args, + isCustomCommand, + }, + 'Parsed pip-compile command from header', + ); + if (options.outputFile) { + // TODO(not7cd): This file path can be relative like `reqs/main.txt` + const file = upath.parse(outputFileName).base; + if (options.outputFile !== file) { + // we don't trust the user-supplied output-file argument; TODO(not7cd): use our value here + logger.warn( + { outputFile: options.outputFile, actualPath: file }, + 'pip-compile was previously executed with an unexpected `--output-file` filename', + ); + } + pipCompileArgs.outputFile = options.outputFile; + } + if (args.length === 0) { + logger.warn('Assuming implicit source file of requirements.in'); + pipCompileArgs.sourceFiles.push('requirements.in'); // implicit + } else { + pipCompileArgs.sourceFiles.push(...args); + } + } catch (error) { + logger.error( + error, + 'Failed to parse pip-compile command from header with commander', + ); + } + return pipCompileArgs; + } + logger.trace({ compileCommand }, 'Failed to parse command'); + return pipCompileArgs; +} diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts new file mode 100644 index 00000000000000..8abae1475cd365 --- /dev/null +++ b/lib/modules/manager/pip-compile/extract.ts @@ -0,0 +1,65 @@ +import { logger } from '../../../logger'; +import { readLocalFile } from '../../../util/fs'; +import { extractPackageFile as extractRequirementsFile } from '../pip_requirements/extract'; +import { extractPackageFile as extractSetupPyFile } from '../pip_setup'; +import { extractPackageFile as extractSetupCfgFile } from '../setup-cfg'; +import type { ExtractConfig, PackageFile } from '../types'; +import { extractHeaderCommand } from './common'; + +export async function extractAllPackageFiles( + config: ExtractConfig, + packageFiles: string[], +): Promise { + const result: PackageFile[] = []; + for (const lockFile of packageFiles) { + logger.debug({ packageFile: lockFile }, 'READING FILE'); + const content = await readLocalFile(lockFile, 'utf8'); + // istanbul ignore else + if (content) { + const pipCompileArgs = extractHeaderCommand(content, lockFile); + // TODO(not7cd): handle locked deps + // const lockedDeps = extractRequirementsFile(content); + for (const sourceFile of pipCompileArgs.sourceFiles) { + const content = await readLocalFile(sourceFile, 'utf8'); + if (content) { + if (sourceFile.endsWith('.in')) { + const deps = extractRequirementsFile(content); + if (deps) { + result.push({ + ...deps, + lockFiles: [lockFile], + packageFile: sourceFile, + }); + } + } else if (sourceFile.endsWith('.py')) { + const deps = extractSetupPyFile(content, sourceFile, config); + if (deps) { + result.push({ + ...deps, + lockFiles: [lockFile], + packageFile: sourceFile, + }); + } + } else if (sourceFile.endsWith('.cfg')) { + const deps = await extractSetupCfgFile(content); + if (deps) { + result.push({ + ...deps, + lockFiles: [lockFile], + packageFile: sourceFile, + }); + } + } else { + // TODO(not7cd): extract based on manager: pep621, etc. + logger.debug({ packageFile: sourceFile }, 'Not supported'); + } + } else { + logger.debug({ packageFile: sourceFile }, 'No content found'); + } + } + } else { + logger.debug({ packageFile: lockFile }, 'No content found'); + } + } + return result; +} diff --git a/lib/modules/manager/pip-compile/index.ts b/lib/modules/manager/pip-compile/index.ts index 98e616010896be..5dc6a3e0275334 100644 --- a/lib/modules/manager/pip-compile/index.ts +++ b/lib/modules/manager/pip-compile/index.ts @@ -2,7 +2,7 @@ import type { Category } from '../../../constants'; import { GitTagsDatasource } from '../../datasource/git-tags'; import { PypiDatasource } from '../../datasource/pypi'; -export { extractPackageFile } from '../pip_requirements/extract'; +export { extractAllPackageFiles } from './extract'; export { updateArtifacts } from './artifacts'; export const supportsLockFileMaintenance = true; diff --git a/lib/modules/manager/pip-compile/readme.md b/lib/modules/manager/pip-compile/readme.md index 09589f039f9ab7..49463ce829c077 100644 --- a/lib/modules/manager/pip-compile/readme.md +++ b/lib/modules/manager/pip-compile/readme.md @@ -12,20 +12,30 @@ You can "activate" the manager by specifying a `fileMatch` pattern such as: ```json { "pip-compile": { - "fileMatch": ["(^|/)requirements\\.in$"] + "fileMatch": ["(^|/)requirements\\.txt$"] } } ``` -### Assumption of `.in`/`.txt` +### Assumption of header with a command -If Renovate matches/extracts a file, it assumes that the corresponding output file is found by swapping the `.in` for `.txt`. -e.g. `requirements.in` => `requirements.txt` -It will not work if files are in separate directories, including `input/requirements.in` and `output/requirements.txt`. +If Renovate matches a `pip-compile` output file it will extract original command that was used to create it from header in this file. Because of that `pip-compile` manager poses restrictions on how this file is generated: -If no `.in` suffix is found, then a `.txt` suffix is appended for the output file, e.g. `foo.file` would look for a corresponding `foo.file.txt`. +- Use default header generation, don't use `--no-header` option. +- Pass all source files explicitly. -We intend to make the mapping configurable in future iterations. +In turn `pip-compile` manager will find all source files and parse them as package files. + +Example header: + +``` +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --no-emit-index-url --output-file=requirements.txt requirements.in +# +``` ### Configuration of Python version @@ -44,8 +54,11 @@ To get Renovate to use another version of Python, add a constraints` rule to the Renovate reads the `requirements.txt` file and extracts these `pip-compile` arguments: -- `--generate-hashes` -- `--allow-unsafe` -- `--no-emit-index-url` -- `--strip-extras` -- `--resolver` +- source files as positional arguments +- `--output-file` + +All other arguments will be passed over without modification. + +#### `CUSTOM_COMPILE_COMMAND` + +If a wrapper is used, it should not obstruct these arguments. From 831a7fd2ddc371193e2c234b8d55ead27cedc406 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Thu, 25 Jan 2024 15:33:41 +0100 Subject: [PATCH 02/85] Comment --- lib/modules/manager/pip-compile/artifacts.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index 86415f4ceff716..7b628b037aab72 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -26,6 +26,7 @@ export async function updateArtifacts({ newPackageFileContent: newInputContent, config, }: UpdateArtifact): Promise { + // TODO(not7cd): must be extracted again or passed from PackageFileContent.lockFiles const outputFileName = inputFileName.replace(regEx(/(\.in)?$/), '.txt'); logger.debug( `pipCompile.updateArtifacts(${inputFileName}->${outputFileName})`, From 26f9873aa7968cd628713ae2ee94a4ad9db1ac28 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Fri, 26 Jan 2024 11:58:38 +0100 Subject: [PATCH 03/85] Fix command parsing --- lib/modules/manager/pip-compile/common.ts | 37 +++++++++++++---------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 8bd749f7da9e1b..a8cbf087abc53a 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -69,13 +69,14 @@ export const allowedPipArguments = [ ]; // as commander.js is already used, we will reuse it's argument parsing capability -const dummyPipCompile = new Command() +const dummyPipCompile = new Command(); +dummyPipCompile + .argument('') .option('-o, --output-file ') // .option('--no-emit-index-url') - .option('--extra-index-url') - // .enablePositionalOptions() - .allowUnknownOption() - .allowExcessArguments(); + .option('--extra-index-url '); +// .allowUnknownOption() +// .allowExcessArguments() interface PipCompileArgs { command: string; @@ -93,9 +94,11 @@ export function extractHeaderCommand( ): PipCompileArgs { const compileCommand = constraintLineRegex.exec(content); if (compileCommand?.groups) { - logger.debug(`Found pip-compile header: ${compileCommand[0]}`); + logger.debug( + `Found pip-compile header in ${outputFileName}: \n${compileCommand[0]}`, + ); } else { - logger.error('Failed to extract command from header'); + logger.error(`Failed to extract command from header in ${outputFileName}`); // TODO(not7cd): throw } const pipCompileArgs: PipCompileArgs = { @@ -109,17 +112,25 @@ export function extractHeaderCommand( if (compileCommand.groups.arguments) { pipCompileArgs.argv.push(...split(compileCommand.groups.arguments)); } + logger.debug( + { argv: pipCompileArgs.argv }, + 'Extracted pip-compile command from header', + ); try { const isCustomCommand = pipCompileArgs.argv[0] !== 'pip-compile'; - const parsedCommand = dummyPipCompile.parse(pipCompileArgs.argv); + const parsedCommand = dummyPipCompile.parse( + // parse is expecting argv[0] to be process.execPath + [''].concat(pipCompileArgs.argv), + ); const options = parsedCommand.opts(); // TODO(not7cd): trace unsupported options - const args = parsedCommand.args; + pipCompileArgs.sourceFiles = parsedCommand.args; logger.debug( { argv: pipCompileArgs.argv, options, - sourceFiles: args, + sourceFiles: pipCompileArgs.sourceFiles, + args: parsedCommand.args, isCustomCommand, }, 'Parsed pip-compile command from header', @@ -136,12 +147,6 @@ export function extractHeaderCommand( } pipCompileArgs.outputFile = options.outputFile; } - if (args.length === 0) { - logger.warn('Assuming implicit source file of requirements.in'); - pipCompileArgs.sourceFiles.push('requirements.in'); // implicit - } else { - pipCompileArgs.sourceFiles.push(...args); - } } catch (error) { logger.error( error, From b90a58c1d6cab00a49d3a0a328988ac45038675e Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Fri, 26 Jan 2024 12:56:36 +0100 Subject: [PATCH 04/85] updateArtifacts now refers to matched lockFiles --- lib/modules/manager/pip-compile/artifacts.ts | 31 +++++++++++++---- lib/modules/manager/pip-compile/common.ts | 4 +-- lib/modules/manager/pip-compile/extract.ts | 35 +++++++++++++++++++- lib/modules/manager/pip-compile/index.ts | 2 +- 4 files changed, 62 insertions(+), 10 deletions(-) diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index 7b628b037aab72..1abba92fbdb2f6 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -8,7 +8,6 @@ import { writeLocalFile, } from '../../../util/fs'; import { getRepoStatus } from '../../../util/git'; -import { regEx } from '../../../util/regex'; import type { UpdateArtifact, UpdateArtifactsResult } from '../types'; import { extractHeaderCommand, getExecOptions } from './common'; @@ -16,18 +15,38 @@ export function constructPipCompileCmd( content: string, outputFileName: string, ): string { - const pipCompileArgs = extractHeaderCommand(content, outputFileName); - // TODO(not7cd): sanitize args that require quotes, .map((argument) => quote(argument)) - return pipCompileArgs.argv.join(' '); + const defaultSourceFile = outputFileName.replace('.txt', '.in'); + try { + const pipCompileArgs = extractHeaderCommand(content, outputFileName); + const newCmd = []; + if (!pipCompileArgs.command || pipCompileArgs.command === '') { + logger.trace('No command detected, assuming pip-compile'); + newCmd.push('pip-compile'); + } + // if (pipCompileArgs.sourceFiles.length === 0) { + // logger.warn('Assuming implicit source file of requirements.in'); + // pipCompileArgs.sourceFiles.push('requirements.in'); // implicit + // pipCompileArgs.argv.push('requirements.in'); // TODO(not7cd): dedup + // } + // TODO(not7cd): sanitize args that require quotes, .map((argument) => quote(argument)) + return pipCompileArgs.argv.join(' '); + } catch (error) { + return `pip-compile ${defaultSourceFile}`; + } } export async function updateArtifacts({ packageFileName: inputFileName, newPackageFileContent: newInputContent, + updatedDeps, config, }: UpdateArtifact): Promise { - // TODO(not7cd): must be extracted again or passed from PackageFileContent.lockFiles - const outputFileName = inputFileName.replace(regEx(/(\.in)?$/), '.txt'); + if (!config.lockFiles) { + logger.error(`No lock files associated with ${inputFileName}`); + return null; + } + // TODO(not7cd): for each + const outputFileName = config.lockFiles[0]; logger.debug( `pipCompile.updateArtifacts(${inputFileName}->${outputFileName})`, ); diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index a8cbf087abc53a..ec2f870b0db76c 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -74,8 +74,8 @@ dummyPipCompile .argument('') .option('-o, --output-file ') // .option('--no-emit-index-url') - .option('--extra-index-url '); -// .allowUnknownOption() + .option('--extra-index-url ') + .allowUnknownOption(); // .allowExcessArguments() interface PipCompileArgs { diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index 8abae1475cd365..35147efb58f040 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -3,9 +3,41 @@ import { readLocalFile } from '../../../util/fs'; import { extractPackageFile as extractRequirementsFile } from '../pip_requirements/extract'; import { extractPackageFile as extractSetupPyFile } from '../pip_setup'; import { extractPackageFile as extractSetupCfgFile } from '../setup-cfg'; -import type { ExtractConfig, PackageFile } from '../types'; +import type { ExtractConfig, PackageFile, PackageFileContent } from '../types'; import { extractHeaderCommand } from './common'; +function matchManager(filename: string): string { + // naive, could be improved + if (filename.endsWith('.in')) { + return 'pip_requirements'; + } + if (filename.endsWith('.py')) { + return 'pip_setup'; + } + if (filename.endsWith('.cfg')) { + return 'setup-cfg'; + } + if (filename.endsWith('.toml')) { + return 'pep621'; + } + return 'unknown'; +} + +export function extractPackageFile( + content: string, + _packageFile: string, + _config: ExtractConfig, +): PackageFileContent | null { + const manager = matchManager(_packageFile); + switch (manager) { + case 'pip_requirements': + return extractRequirementsFile(content); + default: + logger.error(`Unsupported manager ${manager} for ${_packageFile}`); + return null; + } +} + export async function extractAllPackageFiles( config: ExtractConfig, packageFiles: string[], @@ -22,6 +54,7 @@ export async function extractAllPackageFiles( for (const sourceFile of pipCompileArgs.sourceFiles) { const content = await readLocalFile(sourceFile, 'utf8'); if (content) { + // TODO(not7cd): refactor with extractPackageFile if (sourceFile.endsWith('.in')) { const deps = extractRequirementsFile(content); if (deps) { diff --git a/lib/modules/manager/pip-compile/index.ts b/lib/modules/manager/pip-compile/index.ts index 5dc6a3e0275334..b6bc47fca61d1e 100644 --- a/lib/modules/manager/pip-compile/index.ts +++ b/lib/modules/manager/pip-compile/index.ts @@ -2,7 +2,7 @@ import type { Category } from '../../../constants'; import { GitTagsDatasource } from '../../datasource/git-tags'; import { PypiDatasource } from '../../datasource/pypi'; -export { extractAllPackageFiles } from './extract'; +export { extractAllPackageFiles, extractPackageFile } from './extract'; export { updateArtifacts } from './artifacts'; export const supportsLockFileMaintenance = true; From a9ed0994faafc287af6a9b2e8b434b3339ae5215 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Fri, 26 Jan 2024 13:09:03 +0100 Subject: [PATCH 05/85] Add strict arg to constructPipCompileCmd --- lib/modules/manager/pip-compile/artifacts.ts | 4 ++++ lib/modules/manager/pip-compile/common.ts | 7 +++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index 1abba92fbdb2f6..27084280738f59 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -14,10 +14,14 @@ import { extractHeaderCommand, getExecOptions } from './common'; export function constructPipCompileCmd( content: string, outputFileName: string, + strict: boolean = true, ): string { const defaultSourceFile = outputFileName.replace('.txt', '.in'); try { const pipCompileArgs = extractHeaderCommand(content, outputFileName); + if (strict && pipCompileArgs.isCustomCommand) { + logger.error({ command: pipCompileArgs.command }, 'Custom command'); + } const newCmd = []; if (!pipCompileArgs.command || pipCompileArgs.command === '') { logger.trace('No command detected, assuming pip-compile'); diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index ec2f870b0db76c..34eec9a7a37543 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -80,6 +80,7 @@ dummyPipCompile interface PipCompileArgs { command: string; + isCustomCommand: boolean; outputFile?: string; extra?: string[]; constraint?: string[]; @@ -101,9 +102,11 @@ export function extractHeaderCommand( logger.error(`Failed to extract command from header in ${outputFileName}`); // TODO(not7cd): throw } + // TODO(not7cd): construct at return const pipCompileArgs: PipCompileArgs = { argv: [], command: '', + isCustomCommand: false, sourceFiles: [], }; if (compileCommand?.groups) { @@ -117,7 +120,7 @@ export function extractHeaderCommand( 'Extracted pip-compile command from header', ); try { - const isCustomCommand = pipCompileArgs.argv[0] !== 'pip-compile'; + pipCompileArgs.isCustomCommand = pipCompileArgs.argv[0] !== 'pip-compile'; const parsedCommand = dummyPipCompile.parse( // parse is expecting argv[0] to be process.execPath [''].concat(pipCompileArgs.argv), @@ -131,7 +134,7 @@ export function extractHeaderCommand( options, sourceFiles: pipCompileArgs.sourceFiles, args: parsedCommand.args, - isCustomCommand, + isCustomCommand: pipCompileArgs.isCustomCommand, }, 'Parsed pip-compile command from header', ); From 3c46844aee2fdb5e59a0a28f34998b6bc3d663e6 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Fri, 26 Jan 2024 13:20:49 +0100 Subject: [PATCH 06/85] Refactor matchManager --- lib/modules/manager/pip-compile/extract.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index 35147efb58f040..1d6d95e8ae7954 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -7,19 +7,19 @@ import type { ExtractConfig, PackageFile, PackageFileContent } from '../types'; import { extractHeaderCommand } from './common'; function matchManager(filename: string): string { - // naive, could be improved - if (filename.endsWith('.in')) { - return 'pip_requirements'; - } - if (filename.endsWith('.py')) { + if (filename.endsWith('setup.py')) { return 'pip_setup'; } - if (filename.endsWith('.cfg')) { + if (filename.endsWith('setup.cfg')) { return 'setup-cfg'; } - if (filename.endsWith('.toml')) { + if (filename.endsWith('pyproject.toml')) { return 'pep621'; } + // naive, could be improved, pip_requirements.fileMatch ??? + if (filename.endsWith('.in')) { + return 'pip_requirements'; + } return 'unknown'; } From 79d17225fa49dfc38236141028e379c9783c8bc1 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Fri, 26 Jan 2024 13:51:14 +0100 Subject: [PATCH 07/85] Refactor extractAllPackageFiles to use extractPackageFile --- lib/modules/manager/pip-compile/extract.ts | 56 +++++++++------------- 1 file changed, 22 insertions(+), 34 deletions(-) diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index 1d6d95e8ae7954..cb446e022fe997 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -16,20 +16,26 @@ function matchManager(filename: string): string { if (filename.endsWith('pyproject.toml')) { return 'pep621'; } - // naive, could be improved, pip_requirements.fileMatch ??? + // naive, could be improved, maybe use pip_requirements.fileMatch if (filename.endsWith('.in')) { return 'pip_requirements'; } return 'unknown'; } -export function extractPackageFile( +export async function extractPackageFile( content: string, _packageFile: string, _config: ExtractConfig, -): PackageFileContent | null { +): Promise { + logger.trace('pip-compile.extractPackageFile()'); const manager = matchManager(_packageFile); + // TODO(not7cd): extract based on manager: pep621, identify other missing source types switch (manager) { + case 'pip_setup': + return extractSetupPyFile(content, _packageFile, _config); + case 'setup-cfg': + return await extractSetupCfgFile(content); case 'pip_requirements': return extractRequirementsFile(content); default: @@ -42,9 +48,9 @@ export async function extractAllPackageFiles( config: ExtractConfig, packageFiles: string[], ): Promise { + logger.trace('pip-compile.extractAllPackageFiles()'); const result: PackageFile[] = []; for (const lockFile of packageFiles) { - logger.debug({ packageFile: lockFile }, 'READING FILE'); const content = await readLocalFile(lockFile, 'utf8'); // istanbul ignore else if (content) { @@ -54,37 +60,18 @@ export async function extractAllPackageFiles( for (const sourceFile of pipCompileArgs.sourceFiles) { const content = await readLocalFile(sourceFile, 'utf8'); if (content) { - // TODO(not7cd): refactor with extractPackageFile - if (sourceFile.endsWith('.in')) { - const deps = extractRequirementsFile(content); - if (deps) { - result.push({ - ...deps, - lockFiles: [lockFile], - packageFile: sourceFile, - }); - } - } else if (sourceFile.endsWith('.py')) { - const deps = extractSetupPyFile(content, sourceFile, config); - if (deps) { - result.push({ - ...deps, - lockFiles: [lockFile], - packageFile: sourceFile, - }); - } - } else if (sourceFile.endsWith('.cfg')) { - const deps = await extractSetupCfgFile(content); - if (deps) { - result.push({ - ...deps, - lockFiles: [lockFile], - packageFile: sourceFile, - }); - } + const deps = await extractPackageFile(content, sourceFile, config); + if (deps) { + result.push({ + ...deps, + lockFiles: [lockFile], + packageFile: sourceFile, + }); } else { - // TODO(not7cd): extract based on manager: pep621, etc. - logger.debug({ packageFile: sourceFile }, 'Not supported'); + logger.error( + { packageFile: sourceFile }, + 'Failed to extract dependencies', + ); } } else { logger.debug({ packageFile: sourceFile }, 'No content found'); @@ -94,5 +81,6 @@ export async function extractAllPackageFiles( logger.debug({ packageFile: lockFile }, 'No content found'); } } + // TODO(not7cd): sort by requirement layering (-r -c within .in files) return result; } From 81a8a70c7cd7dd9e954206e85373e2f521457d8f Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Fri, 26 Jan 2024 23:54:42 +0100 Subject: [PATCH 08/85] fix updateArtifacts, loop over lockFiles --- lib/modules/manager/pip-compile/artifacts.ts | 76 ++++++++++---------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index 27084280738f59..7bf0fd23c5f1a9 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -49,58 +49,58 @@ export async function updateArtifacts({ logger.error(`No lock files associated with ${inputFileName}`); return null; } - // TODO(not7cd): for each - const outputFileName = config.lockFiles[0]; logger.debug( - `pipCompile.updateArtifacts(${inputFileName}->${outputFileName})`, + `pipCompile.updateArtifacts(${inputFileName}->${JSON.stringify( + config.lockFiles, + )})`, ); - const existingOutput = await readLocalFile(outputFileName, 'utf8'); - if (!existingOutput) { - logger.debug('No pip-compile output file found'); - return null; - } - try { - await writeLocalFile(inputFileName, newInputContent); - // TODO(not7cd): check --rebuild and --upgrade option - if (config.isLockFileMaintenance) { - await deleteLocalFile(outputFileName); - } - const cmd = constructPipCompileCmd(existingOutput, outputFileName); - const execOptions: ExecOptions = await getExecOptions( - config, - inputFileName, - ); - logger.trace({ cmd }, 'pip-compile command'); - await exec(cmd, execOptions); - const status = await getRepoStatus(); - if (!status?.modified.includes(outputFileName)) { + const result: UpdateArtifactsResult[] = []; + for (const outputFileName of config.lockFiles) { + const existingOutput = await readLocalFile(outputFileName, 'utf8'); + if (!existingOutput) { + logger.debug('No pip-compile output file found'); return null; } - logger.debug('Returning updated pip-compile result'); - return [ - { + try { + await writeLocalFile(inputFileName, newInputContent); + // TODO(not7cd): check --rebuild and --upgrade option + if (config.isLockFileMaintenance) { + await deleteLocalFile(outputFileName); + } + const cmd = constructPipCompileCmd(existingOutput, outputFileName); + const execOptions: ExecOptions = await getExecOptions( + config, + inputFileName, + ); + logger.trace({ cmd }, 'pip-compile command'); + await exec(cmd, execOptions); + const status = await getRepoStatus(); + if (!status?.modified.includes(outputFileName)) { + return null; + } + result.push({ file: { type: 'addition', path: outputFileName, contents: await readLocalFile(outputFileName, 'utf8'), }, - }, - ]; - } catch (err) { - // istanbul ignore if - if (err.message === TEMPORARY_ERROR) { - throw err; - } - logger.debug({ err }, 'Failed to pip-compile'); - return [ - { + }); + } catch (err) { + // istanbul ignore if + if (err.message === TEMPORARY_ERROR) { + throw err; + } + logger.debug({ err }, 'Failed to pip-compile'); + result.push({ artifactError: { lockFile: outputFileName, stderr: err.message, }, - }, - ]; + }); + } } + logger.debug('Returning updated pip-compile result'); + return result; } // TODO(not7cd): remove, legacy resolver is deprecated and will be removed From 51d44b7e29e0de1457cd2c03f5e8a5f7e4cd3b13 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Sat, 27 Jan 2024 00:26:26 +0100 Subject: [PATCH 09/85] Throw errors in constructPipCompileCmd --- lib/modules/manager/pip-compile/artifacts.ts | 21 ++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index 7bf0fd23c5f1a9..20100552a655cb 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -19,19 +19,20 @@ export function constructPipCompileCmd( const defaultSourceFile = outputFileName.replace('.txt', '.in'); try { const pipCompileArgs = extractHeaderCommand(content, outputFileName); + if (pipCompileArgs.argv.length === 0) { + throw new Error('Failed to parse extracted command'); + } if (strict && pipCompileArgs.isCustomCommand) { logger.error({ command: pipCompileArgs.command }, 'Custom command'); + throw new Error( + 'Custom command detected, disable strict mode if self-hosted', + ); } - const newCmd = []; - if (!pipCompileArgs.command || pipCompileArgs.command === '') { - logger.trace('No command detected, assuming pip-compile'); - newCmd.push('pip-compile'); + if (pipCompileArgs.sourceFiles.length === 0) { + throw new Error( + 'No source files detected in command, pass at least one package file explicitly', + ); } - // if (pipCompileArgs.sourceFiles.length === 0) { - // logger.warn('Assuming implicit source file of requirements.in'); - // pipCompileArgs.sourceFiles.push('requirements.in'); // implicit - // pipCompileArgs.argv.push('requirements.in'); // TODO(not7cd): dedup - // } // TODO(not7cd): sanitize args that require quotes, .map((argument) => quote(argument)) return pipCompileArgs.argv.join(' '); } catch (error) { @@ -63,7 +64,7 @@ export async function updateArtifacts({ } try { await writeLocalFile(inputFileName, newInputContent); - // TODO(not7cd): check --rebuild and --upgrade option + // TODO(not7cd): use --upgrade option instead deleting if (config.isLockFileMaintenance) { await deleteLocalFile(outputFileName); } From 3e592e92ebc1b6c69d2435c1dbc9e9f8d74bfbb7 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Sat, 27 Jan 2024 01:30:32 +0100 Subject: [PATCH 10/85] Restrict supported command arguments --- lib/modules/manager/pip-compile/common.ts | 77 ++++++++++++++++++++--- 1 file changed, 70 insertions(+), 7 deletions(-) diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 34eec9a7a37543..f128fcdfd4c6ed 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -8,6 +8,7 @@ import { ensureCacheDir } from '../../../util/fs'; import { regEx } from '../../../util/regex'; import type { UpdateArtifactsConfig } from '../types'; +// TODO(not7cd): rename to getPipToolsVersionConstraint, as constraints have their meaning in pip export function getPipToolsConstraint(config: UpdateArtifactsConfig): string { const { constraints = {} } = config; const { pipTools } = constraints; @@ -56,16 +57,26 @@ export async function getExecOptions( }, }; return execOptions; -} // TODO(not7cd): rename to getPipToolsVersionConstraint, as constraints have their meaning in pipexport function extractHeaderCommand(content: string): string { +} export const constraintLineRegex = regEx( /^(#.*?\r?\n)+# {4}(?\S*)(? .*?)?\r?\n/, ); -export const allowedPipArguments = [ + +export const disallowedPipOptions = [ + '--no-header', // header is required by this manager +]; +export const optionsWithArguments = [ + '--output-file', + '--extra', + '--extra-index-url', +]; +export const allowedPipOptions = [ '--allow-unsafe', '--generate-hashes', - '--no-emit-index-url', // handle this!!! + '--no-emit-index-url', // TODO: handle this!!! '--strip-extras', + ...optionsWithArguments, ]; // as commander.js is already used, we will reuse it's argument parsing capability @@ -93,14 +104,16 @@ export function extractHeaderCommand( content: string, outputFileName: string, ): PipCompileArgs { + const strict: boolean = true; // TODO(not7cd): add to function params const compileCommand = constraintLineRegex.exec(content); if (compileCommand?.groups) { logger.debug( `Found pip-compile header in ${outputFileName}: \n${compileCommand[0]}`, ); } else { - logger.error(`Failed to extract command from header in ${outputFileName}`); - // TODO(not7cd): throw + throw new Error( + `Failed to extract command from header in ${outputFileName}`, + ); } // TODO(not7cd): construct at return const pipCompileArgs: PipCompileArgs = { @@ -110,15 +123,33 @@ export function extractHeaderCommand( sourceFiles: [], }; if (compileCommand?.groups) { - pipCompileArgs.argv = [compileCommand.groups.command]; + const command = compileCommand.groups.command; + const argv: string[] = [command]; + const isCustomCommand = command !== 'pip-compile'; + if (strict && isCustomCommand) { + throw new Error( + `Command "${command}" != "pip-compile", header modified or set by CUSTOM_COMPILE_COMMAND`, + ); + } + if (isCustomCommand) { + logger.debug(`Custom command ${command} detected`); + } + // all arguments are optional, TODO(not7cd): decide if require explicit args if (compileCommand.groups.arguments) { - pipCompileArgs.argv.push(...split(compileCommand.groups.arguments)); + argv.push(...split(compileCommand.groups.arguments)); } logger.debug( { argv: pipCompileArgs.argv }, 'Extracted pip-compile command from header', ); + for (const arg of argv) { + throwForDisallowedOption(arg); + throwForNoEqualSignInOptionWithArgument(arg); + if (strict) { + throwForUnknownOption(arg); + } + } try { pipCompileArgs.isCustomCommand = pipCompileArgs.argv[0] !== 'pip-compile'; const parsedCommand = dummyPipCompile.parse( @@ -161,3 +192,35 @@ export function extractHeaderCommand( logger.trace({ compileCommand }, 'Failed to parse command'); return pipCompileArgs; } + +function throwForDisallowedOption(arg: string): void { + for (const disallowedPipOption of disallowedPipOptions) { + if (arg.startsWith(disallowedPipOption)) { + throw new Error( + `Option ${disallowedPipOption} not allowed for this manager`, + ); + } + } +} + +function throwForNoEqualSignInOptionWithArgument(arg: string): void { + for (const option of optionsWithArguments) { + if (arg.startsWith(option) && !arg.startsWith(`${option}=`)) { + throw new Error( + `Option ${option} must have equal sign '=' separating it's argument`, + ); + } + } +} + +function throwForUnknownOption(arg: string): void { + if (!arg.startsWith('-')) { + return; + } + for (const allowedOption of allowedPipOptions) { + if (arg.startsWith(allowedOption)) { + return; + } + } + throw new Error(`Option ${arg} not supported (yet)`); +} From 3e2fb11717a161247b3a99bec6a146236bb516b0 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Sat, 27 Jan 2024 01:40:23 +0100 Subject: [PATCH 11/85] Refactor extractHeaderCommand --- lib/modules/manager/pip-compile/common.ts | 84 +++++++++++------------ 1 file changed, 39 insertions(+), 45 deletions(-) diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index f128fcdfd4c6ed..8afa3b9b6e4554 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -103,7 +103,7 @@ interface PipCompileArgs { export function extractHeaderCommand( content: string, outputFileName: string, -): PipCompileArgs { +): PipCompileArgs | null { const strict: boolean = true; // TODO(not7cd): add to function params const compileCommand = constraintLineRegex.exec(content); if (compileCommand?.groups) { @@ -115,57 +115,45 @@ export function extractHeaderCommand( `Failed to extract command from header in ${outputFileName}`, ); } - // TODO(not7cd): construct at return - const pipCompileArgs: PipCompileArgs = { - argv: [], - command: '', - isCustomCommand: false, - sourceFiles: [], - }; if (compileCommand?.groups) { - const command = compileCommand.groups.command; - const argv: string[] = [command]; - const isCustomCommand = command !== 'pip-compile'; - if (strict && isCustomCommand) { - throw new Error( - `Command "${command}" != "pip-compile", header modified or set by CUSTOM_COMPILE_COMMAND`, - ); - } - if (isCustomCommand) { - logger.debug(`Custom command ${command} detected`); - } + try { + const command = compileCommand.groups.command; + const argv: string[] = [command]; + const isCustomCommand = command !== 'pip-compile'; + if (strict && isCustomCommand) { + throw new Error( + `Command "${command}" != "pip-compile", header modified or set by CUSTOM_COMPILE_COMMAND`, + ); + } + if (isCustomCommand) { + logger.debug(`Custom command ${command} detected`); + } - // all arguments are optional, TODO(not7cd): decide if require explicit args - if (compileCommand.groups.arguments) { - argv.push(...split(compileCommand.groups.arguments)); - } - logger.debug( - { argv: pipCompileArgs.argv }, - 'Extracted pip-compile command from header', - ); - for (const arg of argv) { - throwForDisallowedOption(arg); - throwForNoEqualSignInOptionWithArgument(arg); - if (strict) { - throwForUnknownOption(arg); + // all arguments are optional, TODO(not7cd): decide if require explicit args + if (compileCommand.groups.arguments) { + argv.push(...split(compileCommand.groups.arguments)); } - } - try { - pipCompileArgs.isCustomCommand = pipCompileArgs.argv[0] !== 'pip-compile'; + logger.debug({ argv }, 'Extracted pip-compile command from header'); + for (const arg of argv) { + throwForDisallowedOption(arg); + throwForNoEqualSignInOptionWithArgument(arg); + if (strict) { + throwForUnknownOption(arg); + } + } + const parsedCommand = dummyPipCompile.parse( // parse is expecting argv[0] to be process.execPath - [''].concat(pipCompileArgs.argv), + [''].concat(argv), ); const options = parsedCommand.opts(); - // TODO(not7cd): trace unsupported options - pipCompileArgs.sourceFiles = parsedCommand.args; + const sourceFiles = parsedCommand.args; logger.debug( { - argv: pipCompileArgs.argv, + argv, options, - sourceFiles: pipCompileArgs.sourceFiles, - args: parsedCommand.args, - isCustomCommand: pipCompileArgs.isCustomCommand, + sourceFiles, + isCustomCommand, }, 'Parsed pip-compile command from header', ); @@ -179,7 +167,14 @@ export function extractHeaderCommand( 'pip-compile was previously executed with an unexpected `--output-file` filename', ); } - pipCompileArgs.outputFile = options.outputFile; + const outputFile = options.outputFile; + return { + argv, + command, + isCustomCommand, + outputFile, + sourceFiles, + }; } } catch (error) { logger.error( @@ -187,10 +182,9 @@ export function extractHeaderCommand( 'Failed to parse pip-compile command from header with commander', ); } - return pipCompileArgs; } logger.trace({ compileCommand }, 'Failed to parse command'); - return pipCompileArgs; + return null; } function throwForDisallowedOption(arg: string): void { From c0361037d9025441580a85daca7f33f8d2755b1b Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Sun, 28 Jan 2024 00:47:53 +0100 Subject: [PATCH 12/85] Refactor option check --- lib/modules/manager/pip-compile/common.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 8afa3b9b6e4554..0e6597ad917ab6 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -135,6 +135,10 @@ export function extractHeaderCommand( } logger.debug({ argv }, 'Extracted pip-compile command from header'); for (const arg of argv) { + // TODO(not7cd): check for "--option -- argument" case + if (!arg.startsWith('-')) { + continue; + } throwForDisallowedOption(arg); throwForNoEqualSignInOptionWithArgument(arg); if (strict) { @@ -208,9 +212,6 @@ function throwForNoEqualSignInOptionWithArgument(arg: string): void { } function throwForUnknownOption(arg: string): void { - if (!arg.startsWith('-')) { - return; - } for (const allowedOption of allowedPipOptions) { if (arg.startsWith(allowedOption)) { return; From 8ed2da3547a77b531cc027159fe90f711686442a Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Sun, 28 Jan 2024 01:00:25 +0100 Subject: [PATCH 13/85] Warn for failed command parsing in extract --- lib/modules/manager/pip-compile/extract.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index cb446e022fe997..525d3676b4cab4 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -51,10 +51,14 @@ export async function extractAllPackageFiles( logger.trace('pip-compile.extractAllPackageFiles()'); const result: PackageFile[] = []; for (const lockFile of packageFiles) { - const content = await readLocalFile(lockFile, 'utf8'); + const lockFileContent = await readLocalFile(lockFile, 'utf8'); // istanbul ignore else - if (content) { - const pipCompileArgs = extractHeaderCommand(content, lockFile); + if (lockFileContent) { + const pipCompileArgs = extractHeaderCommand(lockFileContent, lockFile); + if (!pipCompileArgs) { + logger.warn({ lockFile }, 'Failed to parse command in header'); + continue; + } // TODO(not7cd): handle locked deps // const lockedDeps = extractRequirementsFile(content); for (const sourceFile of pipCompileArgs.sourceFiles) { From ee1bb3b625b377cf105de6324c1e50362c0aefcf Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Sun, 28 Jan 2024 01:00:35 +0100 Subject: [PATCH 14/85] Minor refactor --- lib/modules/manager/pip-compile/common.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 0e6597ad917ab6..a13880e0ac3ac3 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -128,8 +128,6 @@ export function extractHeaderCommand( if (isCustomCommand) { logger.debug(`Custom command ${command} detected`); } - - // all arguments are optional, TODO(not7cd): decide if require explicit args if (compileCommand.groups.arguments) { argv.push(...split(compileCommand.groups.arguments)); } @@ -146,10 +144,8 @@ export function extractHeaderCommand( } } - const parsedCommand = dummyPipCompile.parse( - // parse is expecting argv[0] to be process.execPath - [''].concat(argv), - ); + // Commander.parse is expecting argv[0] to be process.execPath, pass empty string as first value + const parsedCommand = dummyPipCompile.parse(['', ...argv]); const options = parsedCommand.opts(); const sourceFiles = parsedCommand.args; logger.debug( From 9ed1bf66b5afcd5c045df88e2eb11187bcfad1d1 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Sun, 28 Jan 2024 01:36:58 +0100 Subject: [PATCH 15/85] Refactor extractHeaderCommand --- lib/modules/manager/pip-compile/artifacts.ts | 8 -- lib/modules/manager/pip-compile/common.ts | 142 ++++++++++--------- lib/modules/manager/pip-compile/extract.ts | 47 +++--- 3 files changed, 96 insertions(+), 101 deletions(-) diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index 20100552a655cb..0f633f94194fed 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -19,20 +19,12 @@ export function constructPipCompileCmd( const defaultSourceFile = outputFileName.replace('.txt', '.in'); try { const pipCompileArgs = extractHeaderCommand(content, outputFileName); - if (pipCompileArgs.argv.length === 0) { - throw new Error('Failed to parse extracted command'); - } if (strict && pipCompileArgs.isCustomCommand) { logger.error({ command: pipCompileArgs.command }, 'Custom command'); throw new Error( 'Custom command detected, disable strict mode if self-hosted', ); } - if (pipCompileArgs.sourceFiles.length === 0) { - throw new Error( - 'No source files detected in command, pass at least one package file explicitly', - ); - } // TODO(not7cd): sanitize args that require quotes, .map((argument) => quote(argument)) return pipCompileArgs.argv.join(' '); } catch (error) { diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index a13880e0ac3ac3..22a18053f723d5 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -103,88 +103,90 @@ interface PipCompileArgs { export function extractHeaderCommand( content: string, outputFileName: string, -): PipCompileArgs | null { +): PipCompileArgs { const strict: boolean = true; // TODO(not7cd): add to function params const compileCommand = constraintLineRegex.exec(content); - if (compileCommand?.groups) { - logger.debug( - `Found pip-compile header in ${outputFileName}: \n${compileCommand[0]}`, - ); - } else { + if (compileCommand?.groups === undefined) { throw new Error( `Failed to extract command from header in ${outputFileName}`, ); - } - if (compileCommand?.groups) { - try { - const command = compileCommand.groups.command; - const argv: string[] = [command]; - const isCustomCommand = command !== 'pip-compile'; - if (strict && isCustomCommand) { - throw new Error( - `Command "${command}" != "pip-compile", header modified or set by CUSTOM_COMPILE_COMMAND`, - ); - } - if (isCustomCommand) { - logger.debug(`Custom command ${command} detected`); - } - if (compileCommand.groups.arguments) { - argv.push(...split(compileCommand.groups.arguments)); + } else { + logger.debug( + `Found pip-compile header in ${outputFileName}: \n${compileCommand[0]}`, + ); + const command = compileCommand.groups.command; + const argv: string[] = [command]; + const isCustomCommand = command !== 'pip-compile'; + if (strict && isCustomCommand) { + throw new Error( + `Command "${command}" != "pip-compile", header modified or set by CUSTOM_COMPILE_COMMAND`, + ); + } + if (isCustomCommand) { + logger.debug(`Custom command ${command} detected`); + } + if (compileCommand.groups.arguments) { + argv.push(...split(compileCommand.groups.arguments)); + } + logger.debug({ argv }, 'Extracted pip-compile command from header'); + for (const arg of argv) { + // TODO(not7cd): check for "--option -- argument" case + if (!arg.startsWith('-')) { + continue; } - logger.debug({ argv }, 'Extracted pip-compile command from header'); - for (const arg of argv) { - // TODO(not7cd): check for "--option -- argument" case - if (!arg.startsWith('-')) { - continue; - } - throwForDisallowedOption(arg); - throwForNoEqualSignInOptionWithArgument(arg); - if (strict) { - throwForUnknownOption(arg); - } + throwForDisallowedOption(arg); + throwForNoEqualSignInOptionWithArgument(arg); + if (strict) { + throwForUnknownOption(arg); } + } - // Commander.parse is expecting argv[0] to be process.execPath, pass empty string as first value - const parsedCommand = dummyPipCompile.parse(['', ...argv]); - const options = parsedCommand.opts(); - const sourceFiles = parsedCommand.args; - logger.debug( - { - argv, - options, - sourceFiles, - isCustomCommand, - }, - 'Parsed pip-compile command from header', + // Commander.parse is expecting argv[0] to be process.execPath, pass empty string as first value + const parsedCommand = dummyPipCompile.parse(['', ...argv]); + const options = parsedCommand.opts(); + const sourceFiles = parsedCommand.args; + logger.debug( + { + argv, + options, + sourceFiles, + isCustomCommand, + }, + 'Parsed pip-compile command from header', + ); + if (sourceFiles.length === 0) { + throw new Error( + 'No source files detected in command, pass at least one package file explicitly', ); - if (options.outputFile) { - // TODO(not7cd): This file path can be relative like `reqs/main.txt` - const file = upath.parse(outputFileName).base; - if (options.outputFile !== file) { - // we don't trust the user-supplied output-file argument; TODO(not7cd): use our value here - logger.warn( - { outputFile: options.outputFile, actualPath: file }, - 'pip-compile was previously executed with an unexpected `--output-file` filename', - ); - } - const outputFile = options.outputFile; - return { - argv, - command, - isCustomCommand, - outputFile, - sourceFiles, - }; + } + if (options.outputFile) { + // TODO(not7cd): This file path can be relative like `reqs/main.txt` + const file = upath.parse(outputFileName).base; + if (options.outputFile !== file) { + // we don't trust the user-supplied output-file argument; TODO(not7cd): use our value here + logger.warn( + { outputFile: options.outputFile, actualPath: file }, + 'pip-compile was previously executed with an unexpected `--output-file` filename', + ); } - } catch (error) { - logger.error( - error, - 'Failed to parse pip-compile command from header with commander', - ); + const outputFile = options.outputFile; + return { + argv, + command, + isCustomCommand, + outputFile, + sourceFiles, + }; } + logger.debug('Implicit output file'); + return { + argv, + command, + isCustomCommand, + outputFile: '', + sourceFiles, + }; } - logger.trace({ compileCommand }, 'Failed to parse command'); - return null; } function throwForDisallowedOption(arg: string): void { diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index 525d3676b4cab4..7215e94da12927 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -54,32 +54,33 @@ export async function extractAllPackageFiles( const lockFileContent = await readLocalFile(lockFile, 'utf8'); // istanbul ignore else if (lockFileContent) { - const pipCompileArgs = extractHeaderCommand(lockFileContent, lockFile); - if (!pipCompileArgs) { - logger.warn({ lockFile }, 'Failed to parse command in header'); - continue; - } - // TODO(not7cd): handle locked deps - // const lockedDeps = extractRequirementsFile(content); - for (const sourceFile of pipCompileArgs.sourceFiles) { - const content = await readLocalFile(sourceFile, 'utf8'); - if (content) { - const deps = await extractPackageFile(content, sourceFile, config); - if (deps) { - result.push({ - ...deps, - lockFiles: [lockFile], - packageFile: sourceFile, - }); + try { + const pipCompileArgs = extractHeaderCommand(lockFileContent, lockFile); + // TODO(not7cd): handle locked deps + // const lockedDeps = extractRequirementsFile(content); + for (const sourceFile of pipCompileArgs.sourceFiles) { + const content = await readLocalFile(sourceFile, 'utf8'); + if (content) { + const deps = await extractPackageFile(content, sourceFile, config); + if (deps) { + result.push({ + ...deps, + lockFiles: [lockFile], + packageFile: sourceFile, + }); + } else { + logger.error( + { packageFile: sourceFile }, + 'Failed to extract dependencies', + ); + } } else { - logger.error( - { packageFile: sourceFile }, - 'Failed to extract dependencies', - ); + logger.debug({ packageFile: sourceFile }, 'No content found'); } - } else { - logger.debug({ packageFile: sourceFile }, 'No content found'); } + } catch (error) { + logger.warn(error, 'Failed to parse pip-compile command from header'); + continue; } } else { logger.debug({ packageFile: lockFile }, 'No content found'); From 59f52dbd76fede91e2c639488b2d691247f3a025 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Sun, 28 Jan 2024 12:00:48 +0100 Subject: [PATCH 16/85] Dont allow for relative outputfile --- lib/modules/manager/pip-compile/common.ts | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 22a18053f723d5..44bedabb5b15e3 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -70,6 +70,7 @@ export const optionsWithArguments = [ '--output-file', '--extra', '--extra-index-url', + '--resolver', ]; export const allowedPipOptions = [ '--allow-unsafe', @@ -159,17 +160,28 @@ export function extractHeaderCommand( 'No source files detected in command, pass at least one package file explicitly', ); } + let outputFile = ''; if (options.outputFile) { // TODO(not7cd): This file path can be relative like `reqs/main.txt` const file = upath.parse(outputFileName).base; - if (options.outputFile !== file) { - // we don't trust the user-supplied output-file argument; TODO(not7cd): use our value here + // const cwd = upath.parse(outputFileName).dir; + if (options.outputFile === file) { + outputFile = options.outputFile; + } else { + // we don't trust the user-supplied output-file argument; + // TODO(not7cd): allow relative paths logger.warn( { outputFile: options.outputFile, actualPath: file }, 'pip-compile was previously executed with an unexpected `--output-file` filename', ); + // TODO(not7cd): this shouldn't be changed in extract function + outputFile = file; + argv.forEach((item, i) => { + if (item.startsWith('--output-file=')) { + argv[i] = `--output-file=${file}`; + } + }); } - const outputFile = options.outputFile; return { argv, command, @@ -183,7 +195,7 @@ export function extractHeaderCommand( argv, command, isCustomCommand, - outputFile: '', + outputFile, sourceFiles, }; } From 0ef5c2e76b9801016c8b9f06f7073f44f0d3e131 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Sun, 28 Jan 2024 12:01:22 +0100 Subject: [PATCH 17/85] Dont return default command --- lib/modules/manager/pip-compile/artifacts.ts | 21 ++++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index 0f633f94194fed..77cfb5d83a351c 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -16,20 +16,15 @@ export function constructPipCompileCmd( outputFileName: string, strict: boolean = true, ): string { - const defaultSourceFile = outputFileName.replace('.txt', '.in'); - try { - const pipCompileArgs = extractHeaderCommand(content, outputFileName); - if (strict && pipCompileArgs.isCustomCommand) { - logger.error({ command: pipCompileArgs.command }, 'Custom command'); - throw new Error( - 'Custom command detected, disable strict mode if self-hosted', - ); - } - // TODO(not7cd): sanitize args that require quotes, .map((argument) => quote(argument)) - return pipCompileArgs.argv.join(' '); - } catch (error) { - return `pip-compile ${defaultSourceFile}`; + const pipCompileArgs = extractHeaderCommand(content, outputFileName); + if (strict && pipCompileArgs.isCustomCommand) { + logger.error({ command: pipCompileArgs.command }, 'Custom command'); + throw new Error( + 'Custom command detected, disable strict mode if self-hosted', + ); } + // TODO(not7cd): sanitize args that require quotes, .map((argument) => quote(argument)) + return pipCompileArgs.argv.join(' '); } export async function updateArtifacts({ From b49257b5a4d1837c72704c5afdbbac830c43c501 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Sun, 28 Jan 2024 12:02:49 +0100 Subject: [PATCH 18/85] Update tests with lockFiles in config --- lib/modules/manager/pip-compile/artifacts.spec.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/modules/manager/pip-compile/artifacts.spec.ts b/lib/modules/manager/pip-compile/artifacts.spec.ts index fbb09816a415aa..0719e9cbd5c6a5 100644 --- a/lib/modules/manager/pip-compile/artifacts.spec.ts +++ b/lib/modules/manager/pip-compile/artifacts.spec.ts @@ -166,7 +166,11 @@ describe('modules/manager/pip-compile/artifacts', () => { packageFileName: 'requirements.in', updatedDeps: [], newPackageFileContent: 'some new content', - config: { ...config, constraints: { python: '3.10.2' } }, + config: { + ...config, + constraints: { python: '3.10.2' }, + lockFiles: ['requirements.txt'], + }, }), ).not.toBeNull(); @@ -191,7 +195,7 @@ describe('modules/manager/pip-compile/artifacts', () => { packageFileName: 'requirements.in', updatedDeps: [], newPackageFileContent: '{}', - config, + config: { ...config, lockFiles: ['requirements.txt'] }, }), ).toEqual([ { @@ -215,7 +219,7 @@ describe('modules/manager/pip-compile/artifacts', () => { packageFileName: 'requirements.in', updatedDeps: [], newPackageFileContent: '{}', - config: lockMaintenanceConfig, + config: { ...lockMaintenanceConfig, lockFiles: ['requirements.txt'] }, }), ).not.toBeNull(); expect(execSnapshots).toMatchObject([ @@ -245,6 +249,7 @@ describe('modules/manager/pip-compile/artifacts', () => { config: { ...config, constraints: { python: '3.10.2', pipTools: '6.13.0' }, + lockFiles: ['requirements.txt'], }, }), ).not.toBeNull(); From 3120701d4ffd6a9c87ec41206d045b24c28ec03a Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Tue, 30 Jan 2024 14:33:25 +0100 Subject: [PATCH 19/85] Fix tests --- .../manager/pip-compile/artifacts.spec.ts | 44 ++++++++++++++----- lib/modules/manager/pip-compile/artifacts.ts | 3 +- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/lib/modules/manager/pip-compile/artifacts.spec.ts b/lib/modules/manager/pip-compile/artifacts.spec.ts index 0719e9cbd5c6a5..dd59a0691dc32e 100644 --- a/lib/modules/manager/pip-compile/artifacts.spec.ts +++ b/lib/modules/manager/pip-compile/artifacts.spec.ts @@ -22,6 +22,13 @@ jest.mock('../../../util/host-rules', () => mockDeep()); jest.mock('../../../util/http'); jest.mock('../../datasource', () => mockDeep()); +const simpleHeader = `# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile requirements.in +#`; + const adminConfig: RepoGlobalConfig = { // `join` fixes Windows CI localDir: join('/tmp/github/some/repo'), @@ -57,22 +64,28 @@ describe('modules/manager/pip-compile/artifacts', () => { packageFileName: 'requirements.in', updatedDeps: [], newPackageFileContent: '', - config, + config: { + ...config, + lockFiles: ['requirements.txt'], + }, }), ).toBeNull(); expect(execSnapshots).toEqual([]); }); it('returns null if unchanged', async () => { - fs.readLocalFile.mockResolvedValueOnce('content'); + fs.readLocalFile.mockResolvedValueOnce(simpleHeader); const execSnapshots = mockExecAll(); - fs.readLocalFile.mockResolvedValueOnce('content'); + fs.readLocalFile.mockResolvedValueOnce('new lock'); expect( await updateArtifacts({ packageFileName: 'requirements.in', updatedDeps: [], newPackageFileContent: 'some new content', - config, + config: { + ...config, + lockFiles: ['requirements.txt'], + }, }), ).toBeNull(); expect(execSnapshots).toMatchObject([ @@ -81,20 +94,24 @@ describe('modules/manager/pip-compile/artifacts', () => { }); it('returns updated requirements.txt', async () => { - fs.readLocalFile.mockResolvedValueOnce('current requirements.txt'); + fs.readLocalFile.mockResolvedValueOnce(simpleHeader); const execSnapshots = mockExecAll(); git.getRepoStatus.mockResolvedValue( partial({ modified: ['requirements.txt'], }), ); - fs.readLocalFile.mockResolvedValueOnce('New requirements.txt'); + fs.readLocalFile.mockResolvedValueOnce('new lock'); expect( await updateArtifacts({ packageFileName: 'requirements.in', updatedDeps: [], newPackageFileContent: 'some new content', - config: { ...config, constraints: { python: '3.7' } }, + config: { + ...config, + constraints: { python: '3.7' }, + lockFiles: ['requirements.txt'], + }, }), ).not.toBeNull(); expect(execSnapshots).toMatchObject([ @@ -114,14 +131,18 @@ describe('modules/manager/pip-compile/artifacts', () => { modified: ['requirements.txt'], }), ); - fs.readLocalFile.mockResolvedValueOnce('new lock'); + fs.readLocalFile.mockResolvedValueOnce(simpleHeader); fs.ensureCacheDir.mockResolvedValueOnce('/tmp/renovate/cache/others/pip'); expect( await updateArtifacts({ packageFileName: 'requirements.in', updatedDeps: [], newPackageFileContent: 'some new content', - config: { ...config, constraints: { python: '3.10.2' } }, + config: { + ...config, + constraints: { python: '3.10.2' }, + lockFiles: ['requirements.txt'], + }, }), ).not.toBeNull(); @@ -160,7 +181,7 @@ describe('modules/manager/pip-compile/artifacts', () => { modified: ['requirements.txt'], }), ); - fs.readLocalFile.mockResolvedValueOnce('new lock'); + fs.readLocalFile.mockResolvedValueOnce(simpleHeader); expect( await updateArtifacts({ packageFileName: 'requirements.in', @@ -206,7 +227,7 @@ describe('modules/manager/pip-compile/artifacts', () => { }); it('returns updated requirements.txt when doing lockfile maintenance', async () => { - fs.readLocalFile.mockResolvedValueOnce('Current requirements.txt'); + fs.readLocalFile.mockResolvedValueOnce(simpleHeader); const execSnapshots = mockExecAll(); git.getRepoStatus.mockResolvedValue( partial({ @@ -228,6 +249,7 @@ describe('modules/manager/pip-compile/artifacts', () => { }); it('uses pip-compile version from config', async () => { + fs.readLocalFile.mockResolvedValueOnce(simpleHeader); GlobalConfig.set(dockerAdminConfig); // pip-tools datasource.getPkgReleases.mockResolvedValueOnce({ diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index 77cfb5d83a351c..791b1e7c5dde76 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -34,8 +34,7 @@ export async function updateArtifacts({ config, }: UpdateArtifact): Promise { if (!config.lockFiles) { - logger.error(`No lock files associated with ${inputFileName}`); - return null; + throw new Error(`No lock files associated with ${inputFileName}`); } logger.debug( `pipCompile.updateArtifacts(${inputFileName}->${JSON.stringify( From 8bbe10d894d7ab279ac7efad96b3a3588b3a8f49 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Tue, 30 Jan 2024 23:38:37 +0100 Subject: [PATCH 20/85] Add tests for common module --- .../manager/pip-compile/common.spec.ts | 86 +++++++++++++++++++ lib/modules/manager/pip-compile/common.ts | 1 + 2 files changed, 87 insertions(+) create mode 100644 lib/modules/manager/pip-compile/common.spec.ts diff --git a/lib/modules/manager/pip-compile/common.spec.ts b/lib/modules/manager/pip-compile/common.spec.ts new file mode 100644 index 00000000000000..f68e7f3dde4796 --- /dev/null +++ b/lib/modules/manager/pip-compile/common.spec.ts @@ -0,0 +1,86 @@ +import { mockDeep } from 'jest-mock-extended'; +import { join } from 'upath'; +import { envMock } from '../../../../test/exec-util'; +import { env } from '../../../../test/util'; +import { GlobalConfig } from '../../../config/global'; +import type { RepoGlobalConfig } from '../../../config/types'; +import * as docker from '../../../util/exec/docker'; +import { extractHeaderCommand } from './common'; + +jest.mock('../../../util/exec/env'); +jest.mock('../../../util/fs'); +jest.mock('../../../util/git'); +jest.mock('../../../util/host-rules', () => mockDeep()); +jest.mock('../../../util/http'); + +function getCommandInHeader(command: string) { + return `# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# ${command} +# +`; +} + +const adminConfig: RepoGlobalConfig = { + // `join` fixes Windows CI + localDir: join('/tmp/github/some/repo'), + cacheDir: join('/tmp/renovate/cache'), + containerbaseDir: join('/tmp/renovate/cache/containerbase'), +}; + +process.env.CONTAINERBASE = 'true'; + +describe('modules/manager/pip-compile/common', () => { + beforeEach(() => { + env.getChildProcessEnv.mockReturnValue({ + ...envMock.basic, + LANG: 'en_US.UTF-8', + LC_ALL: 'en_US', + }); + GlobalConfig.set(adminConfig); + docker.resetPrefetchedImages(); + }); + + describe('extractHeaderCommand()', () => { + it.each([ + '-v', + '--generate-hashes', + '--resolver=backtracking', + '--resolver=legacy', + '--output-file=reqs.txt', + ])('returns object on correct options', (argument: string) => { + expect( + extractHeaderCommand( + getCommandInHeader(`pip-compile ${argument} reqs.in`), + 'reqs.txt', + ), + ).toBeObject(); + }); + + it.each(['--resolver', '--output-file reqs.txt', '--extra = jupyter'])( + 'errors on malformed options with argument', + (argument: string) => { + expect(() => + extractHeaderCommand( + getCommandInHeader(`pip-compile ${argument} reqs.in`), + 'reqs.txt', + ), + ).toThrow(/equal sign/); + }, + ); + + it.each(['--foo', '-x', '--$(curl this)', '--bar=sus'])( + 'errors on unknown options', + (argument: string) => { + expect(() => + extractHeaderCommand( + getCommandInHeader(`pip-compile ${argument} reqs.in`), + 'reqs.txt', + ), + ).toThrow(/not supported/); + }, + ); + }); +}); diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 44bedabb5b15e3..d4f90b269d6304 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -73,6 +73,7 @@ export const optionsWithArguments = [ '--resolver', ]; export const allowedPipOptions = [ + '-v', '--allow-unsafe', '--generate-hashes', '--no-emit-index-url', // TODO: handle this!!! From 5dd3f37a2b996bdffd7cdaec0f21246769804e43 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Tue, 30 Jan 2024 23:50:43 +0100 Subject: [PATCH 21/85] Add more tests --- .../manager/pip-compile/common.spec.ts | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/lib/modules/manager/pip-compile/common.spec.ts b/lib/modules/manager/pip-compile/common.spec.ts index f68e7f3dde4796..25f3356ea41d1b 100644 --- a/lib/modules/manager/pip-compile/common.spec.ts +++ b/lib/modules/manager/pip-compile/common.spec.ts @@ -82,5 +82,26 @@ describe('modules/manager/pip-compile/common', () => { ).toThrow(/not supported/); }, ); + + it.each(['--no-header'])( + 'always errors on not allowed options', + (argument: string) => { + expect(() => + extractHeaderCommand( + getCommandInHeader(`pip-compile ${argument} reqs.in`), + 'reqs.txt', + ), + ).toThrow(/not allowed/); + }, + ); + + test('error when no source files passed as arguments', () => { + expect(() => + extractHeaderCommand( + getCommandInHeader(`pip-compile --extra=color`), + 'reqs.txt', + ), + ).toThrow(/source/); + }); }); }); From 0e3e580096213fa713116b59bbc1b1fcce6e2a33 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Wed, 31 Jan 2024 16:49:17 +0100 Subject: [PATCH 22/85] More tests & fixes --- .../manager/pip-compile/common.spec.ts | 32 ++++++++++++++++++- lib/modules/manager/pip-compile/common.ts | 14 ++++---- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/lib/modules/manager/pip-compile/common.spec.ts b/lib/modules/manager/pip-compile/common.spec.ts index 25f3356ea41d1b..bcf62b53e64519 100644 --- a/lib/modules/manager/pip-compile/common.spec.ts +++ b/lib/modules/manager/pip-compile/common.spec.ts @@ -5,7 +5,7 @@ import { env } from '../../../../test/util'; import { GlobalConfig } from '../../../config/global'; import type { RepoGlobalConfig } from '../../../config/types'; import * as docker from '../../../util/exec/docker'; -import { extractHeaderCommand } from './common'; +import { allowedPipOptions, extractHeaderCommand } from './common'; jest.mock('../../../util/exec/env'); jest.mock('../../../util/fs'); @@ -103,5 +103,35 @@ describe('modules/manager/pip-compile/common', () => { ), ).toThrow(/source/); }); + + test('returned sourceFiles returns all source files', () => { + const exampleSourceFiles = [ + 'requirements.in', + 'reqs/testing.in', + 'base.txt', + './lib/setup.py', + 'pyproject.toml', + ]; + expect( + extractHeaderCommand( + getCommandInHeader( + `pip-compile --extra=color ${exampleSourceFiles.join(' ')}`, + ), + 'reqs.txt', + ).sourceFiles, + ).toEqual(exampleSourceFiles); + }); + + it.each(allowedPipOptions)( + 'returned sourceFiles must not contain options', + (argument: string) => { + const sourceFiles = extractHeaderCommand( + getCommandInHeader(`pip-compile ${argument}=dd reqs.in`), + 'reqs.txt', + ).sourceFiles; + expect(sourceFiles).not.toContainEqual(argument); + expect(sourceFiles).toEqual(['reqs.in']); + }, + ); }); }); diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index d4f90b269d6304..34d98a4bcff1d3 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -84,12 +84,11 @@ export const allowedPipOptions = [ // as commander.js is already used, we will reuse it's argument parsing capability const dummyPipCompile = new Command(); dummyPipCompile - .argument('') - .option('-o, --output-file ') - // .option('--no-emit-index-url') + .argument('[sourceFile...]') // optional, so extractHeaderCommand can throw an explicit error + .option('--output-file ') + .option('--extra ') .option('--extra-index-url ') .allowUnknownOption(); -// .allowExcessArguments() interface PipCompileArgs { command: string; @@ -146,7 +145,10 @@ export function extractHeaderCommand( // Commander.parse is expecting argv[0] to be process.execPath, pass empty string as first value const parsedCommand = dummyPipCompile.parse(['', ...argv]); const options = parsedCommand.opts(); - const sourceFiles = parsedCommand.args; + // workaround, not sure how Commander returns named arguments + const sourceFiles = parsedCommand.args.filter( + (arg) => !arg.startsWith('-'), + ); logger.debug( { argv, @@ -214,7 +216,7 @@ function throwForDisallowedOption(arg: string): void { function throwForNoEqualSignInOptionWithArgument(arg: string): void { for (const option of optionsWithArguments) { - if (arg.startsWith(option) && !arg.startsWith(`${option}=`)) { + if (arg === option) { throw new Error( `Option ${option} must have equal sign '=' separating it's argument`, ); From 384154b0f6999fb681476eac710dd2edae9e148b Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Wed, 31 Jan 2024 17:15:36 +0100 Subject: [PATCH 23/85] Refactor tests that expect option skipping --- .../manager/pip-compile/artifacts.spec.ts | 49 ++++++++----------- 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/lib/modules/manager/pip-compile/artifacts.spec.ts b/lib/modules/manager/pip-compile/artifacts.spec.ts index dd59a0691dc32e..67e1a691c0fc29 100644 --- a/lib/modules/manager/pip-compile/artifacts.spec.ts +++ b/lib/modules/manager/pip-compile/artifacts.spec.ts @@ -300,13 +300,13 @@ describe('modules/manager/pip-compile/artifacts', () => { }); describe('constructPipCompileCmd()', () => { - it('returns default cmd for garbage', () => { - expect( + it('throws for garbage', () => { + expect(() => constructPipCompileCmd( Fixtures.get('requirementsNoHeaders.txt'), 'subdir/requirements.txt', ), - ).toBe('pip-compile requirements.in'); + ).toThrow(/extract/); }); it('returns extracted common arguments (like those featured in the README)', () => { @@ -320,37 +320,30 @@ describe('modules/manager/pip-compile/artifacts', () => { ); }); - it('skips unknown arguments', () => { - expect( + it('throws on unknown arguments', () => { + expect(() => constructPipCompileCmd( Fixtures.get('requirementsWithUnknownArguments.txt'), 'subdir/requirements.txt', ), - ).toBe('pip-compile --generate-hashes requirements.in'); - expect(logger.trace).toHaveBeenCalledWith( - { argument: '--version' }, - 'pip-compile argument is not (yet) supported', - ); - expect(logger.warn).toHaveBeenCalledWith( - { argument: '--resolver=foobar' }, - 'pip-compile was previously executed with an unexpected `--resolver` value', - ); + ).toThrow(/supported/); }); - it('skips exploitable subcommands and files', () => { - expect( - constructPipCompileCmd( - Fixtures.get('requirementsWithExploitingArguments.txt'), - 'subdir/requirements.txt', - ), - ).toBe( - 'pip-compile --generate-hashes --output-file=requirements.txt requirements.in', - ); - expect(logger.warn).toHaveBeenCalledWith( - { argument: '--output-file=/etc/shadow' }, - 'pip-compile was previously executed with an unexpected `--output-file` filename', - ); - }); + // TODO(not7cd): check for explotiable commands + // it('skips exploitable subcommands and files', () => { + // expect( + // constructPipCompileCmd( + // Fixtures.get('requirementsWithExploitingArguments.txt'), + // 'subdir/requirements.txt', + // ), + // ).toBe( + // 'pip-compile --generate-hashes --output-file=requirements.txt requirements.in', + // ); + // expect(logger.warn).toHaveBeenCalledWith( + // { argument: '--output-file=/etc/shadow' }, + // 'pip-compile was previously executed with an unexpected `--output-file` filename', + // ); + // }); }); describe('extractResolver()', () => { From 44f751879d9b9e22bb7c71788d11f5545967bfd9 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Wed, 31 Jan 2024 18:03:13 +0100 Subject: [PATCH 24/85] Change custom command handling Allow extraction and parsing but don't contruct command if custom --- .../__fixtures__/requirementsCustomCommand.txt | 18 ++++++++++++++++++ .../manager/pip-compile/artifacts.spec.ts | 10 ++++++++++ lib/modules/manager/pip-compile/artifacts.ts | 3 +-- lib/modules/manager/pip-compile/common.ts | 5 ----- 4 files changed, 29 insertions(+), 7 deletions(-) create mode 100644 lib/modules/manager/pip-compile/__fixtures__/requirementsCustomCommand.txt diff --git a/lib/modules/manager/pip-compile/__fixtures__/requirementsCustomCommand.txt b/lib/modules/manager/pip-compile/__fixtures__/requirementsCustomCommand.txt new file mode 100644 index 00000000000000..82df200bdde4cd --- /dev/null +++ b/lib/modules/manager/pip-compile/__fixtures__/requirementsCustomCommand.txt @@ -0,0 +1,18 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# ./pip-compile-wrapper requirements.in +# +certifi==2023.11.17 + # via requests +charset-normalizer==3.3.2 + # via requests +idna==3.6 + # via requests +markupsafe==2.1.4 + # via jinja2 +requests==2.31.0 + # via -r requirements.in +urllib3==2.1.0 + # via requests diff --git a/lib/modules/manager/pip-compile/artifacts.spec.ts b/lib/modules/manager/pip-compile/artifacts.spec.ts index 67e1a691c0fc29..a9f2c519dfec11 100644 --- a/lib/modules/manager/pip-compile/artifacts.spec.ts +++ b/lib/modules/manager/pip-compile/artifacts.spec.ts @@ -329,6 +329,16 @@ describe('modules/manager/pip-compile/artifacts', () => { ).toThrow(/supported/); }); + it('throws on custom command when strict', () => { + expect(() => + constructPipCompileCmd( + Fixtures.get('requirementsCustomCommand.txt'), + 'subdir/requirements.txt', + true, + ), + ).toThrow(/custom/); + }); + // TODO(not7cd): check for explotiable commands // it('skips exploitable subcommands and files', () => { // expect( diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index 791b1e7c5dde76..d2d4932d3fa571 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -18,9 +18,8 @@ export function constructPipCompileCmd( ): string { const pipCompileArgs = extractHeaderCommand(content, outputFileName); if (strict && pipCompileArgs.isCustomCommand) { - logger.error({ command: pipCompileArgs.command }, 'Custom command'); throw new Error( - 'Custom command detected, disable strict mode if self-hosted', + 'Detected custom command, header modified or set by CUSTOM_COMPILE_COMMAND', ); } // TODO(not7cd): sanitize args that require quotes, .map((argument) => quote(argument)) diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 34d98a4bcff1d3..9b01f980635736 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -118,11 +118,6 @@ export function extractHeaderCommand( const command = compileCommand.groups.command; const argv: string[] = [command]; const isCustomCommand = command !== 'pip-compile'; - if (strict && isCustomCommand) { - throw new Error( - `Command "${command}" != "pip-compile", header modified or set by CUSTOM_COMPILE_COMMAND`, - ); - } if (isCustomCommand) { logger.debug(`Custom command ${command} detected`); } From ae89f9bc9dcfda005e00ca443ed0ce6e1d4e8ebe Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Wed, 31 Jan 2024 18:16:44 +0100 Subject: [PATCH 25/85] Add test for no lockfiles in config --- .../manager/pip-compile/artifacts.spec.ts | 19 +++++++++++++++++++ lib/modules/manager/pip-compile/artifacts.ts | 6 +++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/lib/modules/manager/pip-compile/artifacts.spec.ts b/lib/modules/manager/pip-compile/artifacts.spec.ts index a9f2c519dfec11..25f9afaba056e7 100644 --- a/lib/modules/manager/pip-compile/artifacts.spec.ts +++ b/lib/modules/manager/pip-compile/artifacts.spec.ts @@ -93,6 +93,25 @@ describe('modules/manager/pip-compile/artifacts', () => { ]); }); + it('returns null if no config.lockFiles', async () => { + fs.readLocalFile.mockResolvedValueOnce(simpleHeader); + fs.readLocalFile.mockResolvedValueOnce('new lock'); + expect( + await updateArtifacts({ + packageFileName: 'requirements.in', + updatedDeps: [], + newPackageFileContent: 'some new content', + config: { + ...config, + }, + }), + ).toBeNull(); + expect(logger.warn).toHaveBeenCalledWith( + { packageFileName: 'requirements.in' }, + 'No lock files associated with a package file', + ); + }); + it('returns updated requirements.txt', async () => { fs.readLocalFile.mockResolvedValueOnce(simpleHeader); const execSnapshots = mockExecAll(); diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index d2d4932d3fa571..26a568803587d8 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -33,7 +33,11 @@ export async function updateArtifacts({ config, }: UpdateArtifact): Promise { if (!config.lockFiles) { - throw new Error(`No lock files associated with ${inputFileName}`); + logger.warn( + { packageFileName: inputFileName }, + 'No lock files associated with a package file', + ); + return null; } logger.debug( `pipCompile.updateArtifacts(${inputFileName}->${JSON.stringify( From e2c356a0898d8cb182214bd651ba8238926bc55e Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Wed, 31 Jan 2024 18:41:12 +0100 Subject: [PATCH 26/85] Test extract --- .../manager/pip-compile/extract.spec.ts | 30 +++++++++++++++++++ lib/modules/manager/pip-compile/extract.ts | 22 +++++++------- 2 files changed, 42 insertions(+), 10 deletions(-) create mode 100644 lib/modules/manager/pip-compile/extract.spec.ts diff --git a/lib/modules/manager/pip-compile/extract.spec.ts b/lib/modules/manager/pip-compile/extract.spec.ts new file mode 100644 index 00000000000000..d0e1157c87a0fc --- /dev/null +++ b/lib/modules/manager/pip-compile/extract.spec.ts @@ -0,0 +1,30 @@ +import { Fixtures } from '../../../../test/fixtures'; +import { extractPackageFile } from './extract'; + +describe('modules/manager/pip-compile/extract', () => { + beforeEach(() => {}); + + describe('extractHeaderCommand()', () => { + it('returns object for requirements.in', () => { + expect( + extractPackageFile( + Fixtures.get('requirementsWithHashes.txt'), + 'requirements.in', + {}, + ), + ).toBeObject(); + }); + + it.each([ + 'random.py', + 'app.cfg', + 'already_locked.txt', + // TODO(not7cd) + 'pyproject.toml', + 'setup.py', + 'setup.cfg', + ])('returns null on not supported package files', (file: string) => { + expect(extractPackageFile('some content', file, {})).toBeNull(); + }); + }); +}); diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index 7215e94da12927..f8651ae89deafa 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -1,8 +1,9 @@ import { logger } from '../../../logger'; import { readLocalFile } from '../../../util/fs'; import { extractPackageFile as extractRequirementsFile } from '../pip_requirements/extract'; -import { extractPackageFile as extractSetupPyFile } from '../pip_setup'; -import { extractPackageFile as extractSetupCfgFile } from '../setup-cfg'; +// TODO(not7cd): enable in the next PR, when this can be properly tested +// import { extractPackageFile as extractSetupPyFile } from '../pip_setup'; +// import { extractPackageFile as extractSetupCfgFile } from '../setup-cfg'; import type { ExtractConfig, PackageFile, PackageFileContent } from '../types'; import { extractHeaderCommand } from './common'; @@ -23,19 +24,20 @@ function matchManager(filename: string): string { return 'unknown'; } -export async function extractPackageFile( +export function extractPackageFile( content: string, _packageFile: string, _config: ExtractConfig, -): Promise { +): PackageFileContent | null { logger.trace('pip-compile.extractPackageFile()'); const manager = matchManager(_packageFile); - // TODO(not7cd): extract based on manager: pep621, identify other missing source types + // TODO(not7cd): extract based on manager: pep621, setupdools, identify other missing source types switch (manager) { - case 'pip_setup': - return extractSetupPyFile(content, _packageFile, _config); - case 'setup-cfg': - return await extractSetupCfgFile(content); + // TODO(not7cd): enable in the next PR, when this can be properly tested + // case 'pip_setup': + // return extractSetupPyFile(content, _packageFile, _config); + // case 'setup-cfg': + // return await extractSetupCfgFile(content); case 'pip_requirements': return extractRequirementsFile(content); default: @@ -61,7 +63,7 @@ export async function extractAllPackageFiles( for (const sourceFile of pipCompileArgs.sourceFiles) { const content = await readLocalFile(sourceFile, 'utf8'); if (content) { - const deps = await extractPackageFile(content, sourceFile, config); + const deps = extractPackageFile(content, sourceFile, config); if (deps) { result.push({ ...deps, From d35fc1c11c59c4def9809eca8944be83f4d42e54 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Thu, 1 Feb 2024 15:46:30 +0100 Subject: [PATCH 27/85] Fix package files with multiple lock files --- .../manager/pip-compile/extract.spec.ts | 53 ++++++++++++++++++- lib/modules/manager/pip-compile/extract.ts | 10 ++-- 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/lib/modules/manager/pip-compile/extract.spec.ts b/lib/modules/manager/pip-compile/extract.spec.ts index d0e1157c87a0fc..f0209eb95ffe1d 100644 --- a/lib/modules/manager/pip-compile/extract.spec.ts +++ b/lib/modules/manager/pip-compile/extract.spec.ts @@ -1,10 +1,24 @@ import { Fixtures } from '../../../../test/fixtures'; -import { extractPackageFile } from './extract'; +import { fs } from '../../../../test/util'; +import { extractAllPackageFiles, extractPackageFile } from './extract'; + +jest.mock('../../../util/fs'); + +function getSimpleRequirementsFile(command: string, deps: string[] = []) { + return `# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# ${command} +# + +${deps.join('\n')}`; +} describe('modules/manager/pip-compile/extract', () => { beforeEach(() => {}); - describe('extractHeaderCommand()', () => { + describe('extractPackageFile()', () => { it('returns object for requirements.in', () => { expect( extractPackageFile( @@ -27,4 +41,39 @@ describe('modules/manager/pip-compile/extract', () => { expect(extractPackageFile('some content', file, {})).toBeNull(); }); }); + + describe('extractAllPackageFiles()', () => { + it('support package file with multiple lock files', () => { + fs.readLocalFile.mockResolvedValueOnce( + getSimpleRequirementsFile( + 'pip-compile --output-file=requirements1.txt requirements.in', + ['foo==1.0.1'], + ), + ); + // requirements.in is parsed only once + fs.readLocalFile.mockResolvedValueOnce('foo>=1.0.0'); + fs.readLocalFile.mockResolvedValueOnce( + getSimpleRequirementsFile( + 'pip-compile --output-file=requirements2.txt requirements.in', + ['foo==1.0.2'], + ), + ); + fs.readLocalFile.mockResolvedValueOnce( + getSimpleRequirementsFile( + 'pip-compile --output-file=requirements3.txt requirements.in', + ['foo==1.0.3'], + ), + ); + + const lockFiles = [ + 'requirements1.txt', + 'requirements2.txt', + 'requirements3.txt', + ]; + return extractAllPackageFiles({}, lockFiles).then((packageFiles) => { + expect(packageFiles).not.toBeNull(); + return expect(packageFiles[0]).toHaveProperty('lockFiles', lockFiles); + }); + }); + }); }); diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index f8651ae89deafa..2852b5aae10b56 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -51,7 +51,7 @@ export async function extractAllPackageFiles( packageFiles: string[], ): Promise { logger.trace('pip-compile.extractAllPackageFiles()'); - const result: PackageFile[] = []; + const result = new Map(); for (const lockFile of packageFiles) { const lockFileContent = await readLocalFile(lockFile, 'utf8'); // istanbul ignore else @@ -61,11 +61,15 @@ export async function extractAllPackageFiles( // TODO(not7cd): handle locked deps // const lockedDeps = extractRequirementsFile(content); for (const sourceFile of pipCompileArgs.sourceFiles) { + if (result.has(sourceFile)) { + result.get(sourceFile)?.lockFiles?.push(lockFile); + continue; + } const content = await readLocalFile(sourceFile, 'utf8'); if (content) { const deps = extractPackageFile(content, sourceFile, config); if (deps) { - result.push({ + result.set(sourceFile, { ...deps, lockFiles: [lockFile], packageFile: sourceFile, @@ -89,5 +93,5 @@ export async function extractAllPackageFiles( } } // TODO(not7cd): sort by requirement layering (-r -c within .in files) - return result; + return Array.from(result.values()); } From 3a51becaeb64ebe4036b267a3771ac54a9802771 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Thu, 1 Feb 2024 15:55:20 +0100 Subject: [PATCH 28/85] Ignore lock files in source files --- lib/modules/manager/pip-compile/extract.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index 2852b5aae10b56..f54d019155a432 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -61,6 +61,14 @@ export async function extractAllPackageFiles( // TODO(not7cd): handle locked deps // const lockedDeps = extractRequirementsFile(content); for (const sourceFile of pipCompileArgs.sourceFiles) { + if (packageFiles.includes(sourceFile)) { + // TODO(not7cd): do something about it + logger.warn( + { sourceFile, lockFile }, + 'lock file acts as source file for another lock file', + ); + continue; + } if (result.has(sourceFile)) { result.get(sourceFile)?.lockFiles?.push(lockFile); continue; From 14def15f551529f7d7ffd83fb685883117d08bbc Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Fri, 2 Feb 2024 01:11:06 +0100 Subject: [PATCH 29/85] Add test for lock files in source files --- .../manager/pip-compile/extract.spec.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/lib/modules/manager/pip-compile/extract.spec.ts b/lib/modules/manager/pip-compile/extract.spec.ts index f0209eb95ffe1d..d648600f6e9662 100644 --- a/lib/modules/manager/pip-compile/extract.spec.ts +++ b/lib/modules/manager/pip-compile/extract.spec.ts @@ -75,5 +75,28 @@ describe('modules/manager/pip-compile/extract', () => { return expect(packageFiles[0]).toHaveProperty('lockFiles', lockFiles); }); }); + + it('no lock files in returned package files', () => { + fs.readLocalFile.mockResolvedValueOnce( + getSimpleRequirementsFile('pip-compile --output-file=foo.txt foo.in', [ + 'foo==1.0.1', + ]), + ); + fs.readLocalFile.mockResolvedValueOnce('foo>=1.0.0'); + fs.readLocalFile.mockResolvedValueOnce( + getSimpleRequirementsFile( + 'pip-compile --output-file=bar.txt bar.in foo.txt', + ['foo==1.0.1', 'bar==2.0.0'], + ), + ); + fs.readLocalFile.mockResolvedValueOnce('bar>=1.0.0'); + + const lockFiles = ['foo.txt', 'bar.txt']; + return extractAllPackageFiles({}, lockFiles).then((packageFiles) => { + packageFiles.forEach((packageFile) => { + expect(packageFile).not.toHaveProperty('packageFile', 'foo.txt'); + }); + }); + }); }); }); From 8e1b8fce35503e6a56d5469cfb7945fb208f37aa Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Fri, 2 Feb 2024 01:11:24 +0100 Subject: [PATCH 30/85] Refactor extract --- lib/modules/manager/pip-compile/extract.ts | 83 +++++++++++----------- 1 file changed, 42 insertions(+), 41 deletions(-) diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index f54d019155a432..40a50eefe4e681 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -55,49 +55,50 @@ export async function extractAllPackageFiles( for (const lockFile of packageFiles) { const lockFileContent = await readLocalFile(lockFile, 'utf8'); // istanbul ignore else - if (lockFileContent) { - try { - const pipCompileArgs = extractHeaderCommand(lockFileContent, lockFile); - // TODO(not7cd): handle locked deps - // const lockedDeps = extractRequirementsFile(content); - for (const sourceFile of pipCompileArgs.sourceFiles) { - if (packageFiles.includes(sourceFile)) { - // TODO(not7cd): do something about it - logger.warn( - { sourceFile, lockFile }, - 'lock file acts as source file for another lock file', - ); - continue; - } - if (result.has(sourceFile)) { - result.get(sourceFile)?.lockFiles?.push(lockFile); - continue; - } - const content = await readLocalFile(sourceFile, 'utf8'); - if (content) { - const deps = extractPackageFile(content, sourceFile, config); - if (deps) { - result.set(sourceFile, { - ...deps, - lockFiles: [lockFile], - packageFile: sourceFile, - }); - } else { - logger.error( - { packageFile: sourceFile }, - 'Failed to extract dependencies', - ); - } - } else { - logger.debug({ packageFile: sourceFile }, 'No content found'); - } + if (!lockFileContent) { + logger.debug({ lockFile }, 'No content found'); + continue; + } + try { + const pipCompileArgs = extractHeaderCommand(lockFileContent, lockFile); + // TODO(not7cd): handle locked deps + // const lockedDeps = extractRequirementsFile(content); + for (const sourceFile of pipCompileArgs.sourceFiles) { + if (packageFiles.includes(sourceFile)) { + // TODO(not7cd): do something about it + logger.warn( + { sourceFile, lockFile }, + 'lock file acts as source file for another lock file', + ); + continue; + } + if (result.has(sourceFile)) { + result.get(sourceFile)?.lockFiles?.push(lockFile); + continue; + } + const content = await readLocalFile(sourceFile, 'utf8'); + if (!content) { + logger.debug({ sourceFile }, 'No content found'); + continue; + } + + const deps = extractPackageFile(content, sourceFile, config); + if (deps) { + result.set(sourceFile, { + ...deps, + lockFiles: [lockFile], + packageFile: sourceFile, + }); + } else { + logger.error( + { packageFile: sourceFile }, + 'Failed to extract dependencies', + ); } - } catch (error) { - logger.warn(error, 'Failed to parse pip-compile command from header'); - continue; } - } else { - logger.debug({ packageFile: lockFile }, 'No content found'); + } catch (error) { + logger.warn(error, 'Failed to parse pip-compile command from header'); + continue; } } // TODO(not7cd): sort by requirement layering (-r -c within .in files) From 91491e51c3b23b917fd444ab27d7146f73794ed1 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Fri, 2 Feb 2024 01:14:51 +0100 Subject: [PATCH 31/85] eslint --- lib/modules/manager/pip-compile/extract.spec.ts | 2 +- lib/modules/manager/pip-compile/extract.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/modules/manager/pip-compile/extract.spec.ts b/lib/modules/manager/pip-compile/extract.spec.ts index d648600f6e9662..fc854d44cc7532 100644 --- a/lib/modules/manager/pip-compile/extract.spec.ts +++ b/lib/modules/manager/pip-compile/extract.spec.ts @@ -93,7 +93,7 @@ describe('modules/manager/pip-compile/extract', () => { const lockFiles = ['foo.txt', 'bar.txt']; return extractAllPackageFiles({}, lockFiles).then((packageFiles) => { - packageFiles.forEach((packageFile) => { + return packageFiles.forEach((packageFile) => { expect(packageFile).not.toHaveProperty('packageFile', 'foo.txt'); }); }); diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index 40a50eefe4e681..9bd6607a3b0a95 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -54,7 +54,6 @@ export async function extractAllPackageFiles( const result = new Map(); for (const lockFile of packageFiles) { const lockFileContent = await readLocalFile(lockFile, 'utf8'); - // istanbul ignore else if (!lockFileContent) { logger.debug({ lockFile }, 'No content found'); continue; From 77a054f56a1d66cdf544606785340c66cf299f18 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Fri, 2 Feb 2024 01:27:26 +0100 Subject: [PATCH 32/85] refactor try catch --- lib/modules/manager/pip-compile/extract.ts | 75 +++++++++++----------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index 9bd6607a3b0a95..fff3e55d75193a 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -31,7 +31,7 @@ export function extractPackageFile( ): PackageFileContent | null { logger.trace('pip-compile.extractPackageFile()'); const manager = matchManager(_packageFile); - // TODO(not7cd): extract based on manager: pep621, setupdools, identify other missing source types + // TODO(not7cd): extract based on manager: pep621, setuptools, identify other missing source types switch (manager) { // TODO(not7cd): enable in the next PR, when this can be properly tested // case 'pip_setup': @@ -58,47 +58,48 @@ export async function extractAllPackageFiles( logger.debug({ lockFile }, 'No content found'); continue; } + let pipCompileArgs; try { - const pipCompileArgs = extractHeaderCommand(lockFileContent, lockFile); - // TODO(not7cd): handle locked deps - // const lockedDeps = extractRequirementsFile(content); - for (const sourceFile of pipCompileArgs.sourceFiles) { - if (packageFiles.includes(sourceFile)) { - // TODO(not7cd): do something about it - logger.warn( - { sourceFile, lockFile }, - 'lock file acts as source file for another lock file', - ); - continue; - } - if (result.has(sourceFile)) { - result.get(sourceFile)?.lockFiles?.push(lockFile); - continue; - } - const content = await readLocalFile(sourceFile, 'utf8'); - if (!content) { - logger.debug({ sourceFile }, 'No content found'); - continue; - } - - const deps = extractPackageFile(content, sourceFile, config); - if (deps) { - result.set(sourceFile, { - ...deps, - lockFiles: [lockFile], - packageFile: sourceFile, - }); - } else { - logger.error( - { packageFile: sourceFile }, - 'Failed to extract dependencies', - ); - } - } + pipCompileArgs = extractHeaderCommand(lockFileContent, lockFile); } catch (error) { logger.warn(error, 'Failed to parse pip-compile command from header'); continue; } + // TODO(not7cd): handle locked deps + // const lockedDeps = extractRequirementsFile(content); + for (const sourceFile of pipCompileArgs.sourceFiles) { + if (packageFiles.includes(sourceFile)) { + // TODO(not7cd): do something about it + logger.warn( + { sourceFile, lockFile }, + 'lock file acts as source file for another lock file', + ); + continue; + } + if (result.has(sourceFile)) { + result.get(sourceFile)?.lockFiles?.push(lockFile); + continue; + } + const content = await readLocalFile(sourceFile, 'utf8'); + if (!content) { + logger.debug({ sourceFile }, 'No content found'); + continue; + } + + const deps = extractPackageFile(content, sourceFile, config); + if (deps) { + result.set(sourceFile, { + ...deps, + lockFiles: [lockFile], + packageFile: sourceFile, + }); + } else { + logger.warn( + { packageFile: sourceFile }, + 'Failed to extract dependencies', + ); + } + } } // TODO(not7cd): sort by requirement layering (-r -c within .in files) return Array.from(result.values()); From 11b11ca827de9eab40a9abfcc6cb826b3dbff555 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Fri, 2 Feb 2024 01:40:19 +0100 Subject: [PATCH 33/85] test for malformed files --- .../manager/pip-compile/extract.spec.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/lib/modules/manager/pip-compile/extract.spec.ts b/lib/modules/manager/pip-compile/extract.spec.ts index fc854d44cc7532..39c0b5f466cab2 100644 --- a/lib/modules/manager/pip-compile/extract.spec.ts +++ b/lib/modules/manager/pip-compile/extract.spec.ts @@ -99,4 +99,23 @@ describe('modules/manager/pip-compile/extract', () => { }); }); }); + + it('return nothing for malformed files', () => { + fs.readLocalFile.mockResolvedValueOnce( + Fixtures.get('requirementsNoHeaders.txt'), + ); + fs.readLocalFile.mockResolvedValueOnce( + getSimpleRequirementsFile( + 'pip-compile --output-file=foo.txt malformed.in empty.in', + ['foo==1.0.1'], + ), + ); + fs.readLocalFile.mockResolvedValueOnce('!@#$'); + fs.readLocalFile.mockResolvedValueOnce(''); + + const lockFiles = ['foo.txt', 'bar.txt']; + return extractAllPackageFiles({}, lockFiles).then((packageFiles) => { + return expect(packageFiles).toBeEmptyArray(); + }); + }); }); From 45b54903efbfdff6087f0a7d8758c49cea26fe08 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Fri, 2 Feb 2024 01:43:19 +0100 Subject: [PATCH 34/85] test: Add empty lockfile --- lib/modules/manager/pip-compile/extract.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/modules/manager/pip-compile/extract.spec.ts b/lib/modules/manager/pip-compile/extract.spec.ts index 39c0b5f466cab2..57cdbcace6fd55 100644 --- a/lib/modules/manager/pip-compile/extract.spec.ts +++ b/lib/modules/manager/pip-compile/extract.spec.ts @@ -101,6 +101,7 @@ describe('modules/manager/pip-compile/extract', () => { }); it('return nothing for malformed files', () => { + fs.readLocalFile.mockResolvedValueOnce(''); fs.readLocalFile.mockResolvedValueOnce( Fixtures.get('requirementsNoHeaders.txt'), ); @@ -113,7 +114,7 @@ describe('modules/manager/pip-compile/extract', () => { fs.readLocalFile.mockResolvedValueOnce('!@#$'); fs.readLocalFile.mockResolvedValueOnce(''); - const lockFiles = ['foo.txt', 'bar.txt']; + const lockFiles = ['empty.txt', 'noHeader.txt', 'badSource.txt']; return extractAllPackageFiles({}, lockFiles).then((packageFiles) => { return expect(packageFiles).toBeEmptyArray(); }); From 09f0878a4b869292f755fd44b1aaac26cfcadadf Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Fri, 2 Feb 2024 08:10:24 +0100 Subject: [PATCH 35/85] refactoring --- lib/modules/manager/pip-compile/common.ts | 154 ++++++++++----------- lib/modules/manager/pip-compile/extract.ts | 67 +++++---- 2 files changed, 114 insertions(+), 107 deletions(-) diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 9b01f980635736..801ebb5e3d7723 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -103,100 +103,92 @@ interface PipCompileArgs { // TODO(not7cd): test on all correct headers, even with CUSTOM_COMPILE_COMMAND export function extractHeaderCommand( content: string, - outputFileName: string, + fileName: string, ): PipCompileArgs { const strict: boolean = true; // TODO(not7cd): add to function params const compileCommand = constraintLineRegex.exec(content); if (compileCommand?.groups === undefined) { - throw new Error( - `Failed to extract command from header in ${outputFileName}`, - ); - } else { + throw new Error(`Failed to extract command from header in ${fileName}`); + } + logger.debug( + `pip-compile: found header in ${fileName}: \n${compileCommand[0]}`, + ); + const command = compileCommand.groups.command; + const argv: string[] = [command]; + const isCustomCommand = command !== 'pip-compile'; + if (isCustomCommand) { logger.debug( - `Found pip-compile header in ${outputFileName}: \n${compileCommand[0]}`, + `pip-compile: custom command ${command} detected (${fileName})`, ); - const command = compileCommand.groups.command; - const argv: string[] = [command]; - const isCustomCommand = command !== 'pip-compile'; - if (isCustomCommand) { - logger.debug(`Custom command ${command} detected`); - } - if (compileCommand.groups.arguments) { - argv.push(...split(compileCommand.groups.arguments)); + } + if (compileCommand.groups.arguments) { + argv.push(...split(compileCommand.groups.arguments)); + } + logger.debug( + `pip-compile: extracted command from header: ${JSON.stringify(argv)}`, + ); + for (const arg of argv) { + // TODO(not7cd): check for "--option -- argument" case + if (!arg.startsWith('-')) { + continue; } - logger.debug({ argv }, 'Extracted pip-compile command from header'); - for (const arg of argv) { - // TODO(not7cd): check for "--option -- argument" case - if (!arg.startsWith('-')) { - continue; - } - throwForDisallowedOption(arg); - throwForNoEqualSignInOptionWithArgument(arg); - if (strict) { - throwForUnknownOption(arg); - } + throwForDisallowedOption(arg); + throwForNoEqualSignInOptionWithArgument(arg); + if (strict) { + throwForUnknownOption(arg); } + } - // Commander.parse is expecting argv[0] to be process.execPath, pass empty string as first value - const parsedCommand = dummyPipCompile.parse(['', ...argv]); - const options = parsedCommand.opts(); - // workaround, not sure how Commander returns named arguments - const sourceFiles = parsedCommand.args.filter( - (arg) => !arg.startsWith('-'), - ); - logger.debug( - { - argv, - options, - sourceFiles, - isCustomCommand, - }, - 'Parsed pip-compile command from header', + // Commander.parse is expecting argv[0] to be process.execPath, pass empty string as first value + const parsedCommand = dummyPipCompile.parse(['', ...argv]); + const options = parsedCommand.opts(); + // workaround, not sure how Commander returns named arguments + const sourceFiles = parsedCommand.args.filter((arg) => !arg.startsWith('-')); + logger.trace( + { + argv, + options, + sourceFiles, + isCustomCommand, + }, + 'Parsed pip-compile command from header', + ); + if (sourceFiles.length === 0) { + throw new Error( + 'No source files detected in command, pass at least one package file explicitly', ); - if (sourceFiles.length === 0) { - throw new Error( - 'No source files detected in command, pass at least one package file explicitly', + } + let outputFile = ''; + if (options.outputFile) { + // TODO(not7cd): This file path can be relative like `reqs/main.txt` + const file = upath.parse(fileName).base; + if (options.outputFile === file) { + outputFile = options.outputFile; + } else { + // we don't trust the user-supplied output-file argument; + // TODO(not7cd): allow relative paths + logger.warn( + { outputFile: options.outputFile, actualPath: file }, + 'pip-compile was previously executed with an unexpected `--output-file` filename', ); + // TODO(not7cd): this shouldn't be changed in extract function + outputFile = file; + argv.forEach((item, i) => { + if (item.startsWith('--output-file=')) { + argv[i] = `--output-file=${file}`; + } + }); } - let outputFile = ''; - if (options.outputFile) { - // TODO(not7cd): This file path can be relative like `reqs/main.txt` - const file = upath.parse(outputFileName).base; - // const cwd = upath.parse(outputFileName).dir; - if (options.outputFile === file) { - outputFile = options.outputFile; - } else { - // we don't trust the user-supplied output-file argument; - // TODO(not7cd): allow relative paths - logger.warn( - { outputFile: options.outputFile, actualPath: file }, - 'pip-compile was previously executed with an unexpected `--output-file` filename', - ); - // TODO(not7cd): this shouldn't be changed in extract function - outputFile = file; - argv.forEach((item, i) => { - if (item.startsWith('--output-file=')) { - argv[i] = `--output-file=${file}`; - } - }); - } - return { - argv, - command, - isCustomCommand, - outputFile, - sourceFiles, - }; - } - logger.debug('Implicit output file'); - return { - argv, - command, - isCustomCommand, - outputFile, - sourceFiles, - }; + } else { + logger.debug(`pip-compile: implicit output file (${fileName})`); } + return { + argv, + command, + isCustomCommand, + outputFile, + sourceFiles, + }; } function throwForDisallowedOption(arg: string): void { diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index fff3e55d75193a..a6cb2c8fa7fe9d 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -26,11 +26,11 @@ function matchManager(filename: string): string { export function extractPackageFile( content: string, - _packageFile: string, + packageFile: string, _config: ExtractConfig, ): PackageFileContent | null { logger.trace('pip-compile.extractPackageFile()'); - const manager = matchManager(_packageFile); + const manager = matchManager(packageFile); // TODO(not7cd): extract based on manager: pep621, setuptools, identify other missing source types switch (manager) { // TODO(not7cd): enable in the next PR, when this can be properly tested @@ -40,67 +40,82 @@ export function extractPackageFile( // return await extractSetupCfgFile(content); case 'pip_requirements': return extractRequirementsFile(content); + case 'unknown': + logger.warn( + { packageFile }, + `pip-compile: could not determine manager for source file`, + ); + return null; default: - logger.error(`Unsupported manager ${manager} for ${_packageFile}`); + logger.warn( + { packageFile, manager }, + `pip-compile: support for manager is not yet implemented`, + ); return null; } } export async function extractAllPackageFiles( config: ExtractConfig, - packageFiles: string[], + fileMatches: string[], ): Promise { logger.trace('pip-compile.extractAllPackageFiles()'); - const result = new Map(); - for (const lockFile of packageFiles) { - const lockFileContent = await readLocalFile(lockFile, 'utf8'); - if (!lockFileContent) { - logger.debug({ lockFile }, 'No content found'); + const packageFiles = new Map(); + for (const fileMatch of fileMatches) { + const fileContent = await readLocalFile(fileMatch, 'utf8'); + if (!fileContent) { + logger.debug(`pip-compile: no content found for fileMatch ${fileMatch}`); continue; } let pipCompileArgs; try { - pipCompileArgs = extractHeaderCommand(lockFileContent, lockFile); + pipCompileArgs = extractHeaderCommand(fileContent, fileMatch); } catch (error) { - logger.warn(error, 'Failed to parse pip-compile command from header'); + logger.warn( + { fileMatch, error }, + 'Failed to parse pip-compile command from header', + ); continue; } // TODO(not7cd): handle locked deps // const lockedDeps = extractRequirementsFile(content); - for (const sourceFile of pipCompileArgs.sourceFiles) { - if (packageFiles.includes(sourceFile)) { + for (const packageFile of pipCompileArgs.sourceFiles) { + if (fileMatches.includes(packageFile)) { // TODO(not7cd): do something about it logger.warn( - { sourceFile, lockFile }, - 'lock file acts as source file for another lock file', + { sourceFile: packageFile, lockFile: fileMatch }, + 'pip-compile: lock file acts as source file for another lock file', ); continue; } - if (result.has(sourceFile)) { - result.get(sourceFile)?.lockFiles?.push(lockFile); + if (packageFiles.has(packageFile)) { + logger.debug( + `pip-compile: ${packageFile} used in multiple output files`, + ); + packageFiles.get(packageFile)?.lockFiles?.push(fileMatch); continue; } - const content = await readLocalFile(sourceFile, 'utf8'); + const content = await readLocalFile(packageFile, 'utf8'); if (!content) { - logger.debug({ sourceFile }, 'No content found'); + logger.debug(`pip-compile: No content for source file ${packageFile}`); continue; } - const deps = extractPackageFile(content, sourceFile, config); + const deps = extractPackageFile(content, packageFile, config); if (deps) { - result.set(sourceFile, { + packageFiles.set(packageFile, { ...deps, - lockFiles: [lockFile], - packageFile: sourceFile, + lockFiles: [fileMatch], + packageFile, }); } else { logger.warn( - { packageFile: sourceFile }, - 'Failed to extract dependencies', + { packageFile }, + 'pip-compile: failed to find dependencies in source file', ); } } } // TODO(not7cd): sort by requirement layering (-r -c within .in files) - return Array.from(result.values()); + return Array.from(packageFiles.values()); } From 8fffe74ddb0aeb38a2c290f9dcf5162ceb30ea95 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Fri, 2 Feb 2024 11:12:16 +0100 Subject: [PATCH 36/85] Update readme --- lib/modules/manager/pip-compile/readme.md | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/lib/modules/manager/pip-compile/readme.md b/lib/modules/manager/pip-compile/readme.md index 49463ce829c077..bd7cf553c4a292 100644 --- a/lib/modules/manager/pip-compile/readme.md +++ b/lib/modules/manager/pip-compile/readme.md @@ -24,7 +24,7 @@ If Renovate matches a `pip-compile` output file it will extract original command - Use default header generation, don't use `--no-header` option. - Pass all source files explicitly. -In turn `pip-compile` manager will find all source files and parse them as package files. +In turn `pip-compile` manager will find all source files and parse them as package files. Currently only `*.in` files associated with `pip_requirements` manager are handled. Example header: @@ -37,6 +37,19 @@ Example header: # ``` +### Conflicts with other managers + +Because `pip-compile` will update source files with their associated manager you should disable them to avoid running these managers twice. + +```json +{ + "pip_requirements": { + "enabled": false + } +} + +``` + ### Configuration of Python version By default Renovate uses the latest version of Python. @@ -57,8 +70,4 @@ Renovate reads the `requirements.txt` file and extracts these `pip-compile` argu - source files as positional arguments - `--output-file` -All other arguments will be passed over without modification. - -#### `CUSTOM_COMPILE_COMMAND` - -If a wrapper is used, it should not obstruct these arguments. +All other allowed `pip-compile` arguments will be passed over without modification. From 7b8f404b9c78b5107c3d333c73d3c324314e535f Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Fri, 2 Feb 2024 11:19:08 +0100 Subject: [PATCH 37/85] fix --- lib/modules/manager/pip-compile/readme.md | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/modules/manager/pip-compile/readme.md b/lib/modules/manager/pip-compile/readme.md index bd7cf553c4a292..9a3f2ee393753c 100644 --- a/lib/modules/manager/pip-compile/readme.md +++ b/lib/modules/manager/pip-compile/readme.md @@ -47,7 +47,6 @@ Because `pip-compile` will update source files with their associated manager you "enabled": false } } - ``` ### Configuration of Python version From 968754e6a70b22bb1090dd6a2a4ac95d54659c37 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Fri, 2 Feb 2024 14:02:18 +0100 Subject: [PATCH 38/85] Remove empty file --- lib/modules/manager/pip-compile/update.ts | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 lib/modules/manager/pip-compile/update.ts diff --git a/lib/modules/manager/pip-compile/update.ts b/lib/modules/manager/pip-compile/update.ts deleted file mode 100644 index e69de29bb2d1d6..00000000000000 From a1d1c35bd7d77ee01850f82f3c3620d2b548a2da Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Mon, 5 Feb 2024 18:09:52 +0100 Subject: [PATCH 39/85] Quote args Co-authored-by: Michael Kriese --- lib/modules/manager/pip-compile/artifacts.ts | 3 +-- lib/modules/manager/pip-compile/common.ts | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index 26a568803587d8..d04fd1cdc06854 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -22,8 +22,7 @@ export function constructPipCompileCmd( 'Detected custom command, header modified or set by CUSTOM_COMPILE_COMMAND', ); } - // TODO(not7cd): sanitize args that require quotes, .map((argument) => quote(argument)) - return pipCompileArgs.argv.join(' '); + return pipCompileArgs.argv.map(quote).join(' '); } export async function updateArtifacts({ diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 801ebb5e3d7723..a5163b3cddc488 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -175,7 +175,7 @@ export function extractHeaderCommand( outputFile = file; argv.forEach((item, i) => { if (item.startsWith('--output-file=')) { - argv[i] = `--output-file=${file}`; + argv[i] = `--output-file=${quote(file)}`; } }); } From 36685d00ce39a47668651532e64185cf262b146f Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Mon, 5 Feb 2024 18:10:21 +0100 Subject: [PATCH 40/85] Remove unused beforeEach Co-authored-by: Michael Kriese --- lib/modules/manager/pip-compile/common.spec.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/lib/modules/manager/pip-compile/common.spec.ts b/lib/modules/manager/pip-compile/common.spec.ts index bcf62b53e64519..2527998e888fca 100644 --- a/lib/modules/manager/pip-compile/common.spec.ts +++ b/lib/modules/manager/pip-compile/common.spec.ts @@ -33,15 +33,6 @@ const adminConfig: RepoGlobalConfig = { process.env.CONTAINERBASE = 'true'; describe('modules/manager/pip-compile/common', () => { - beforeEach(() => { - env.getChildProcessEnv.mockReturnValue({ - ...envMock.basic, - LANG: 'en_US.UTF-8', - LC_ALL: 'en_US', - }); - GlobalConfig.set(adminConfig); - docker.resetPrefetchedImages(); - }); describe('extractHeaderCommand()', () => { it.each([ From 02b711360bc326f06091a4cee25d8dc8e53fd91d Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Mon, 5 Feb 2024 18:11:53 +0100 Subject: [PATCH 41/85] Import quote --- lib/modules/manager/pip-compile/common.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index a5163b3cddc488..37b9bf5832f6ea 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -1,6 +1,6 @@ import is from '@sindresorhus/is'; import { Command } from 'commander'; -import { split } from 'shlex'; +import { quote, split } from 'shlex'; import upath from 'upath'; import { logger } from '../../../logger'; import type { ExecOptions } from '../../../util/exec/types'; From 9319d7d912ec5d62217335c63bb71a6d887784d1 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Mon, 5 Feb 2024 18:12:42 +0100 Subject: [PATCH 42/85] Revert code move --- lib/modules/manager/pip-compile/common.ts | 50 +++++++++++------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 37b9bf5832f6ea..c9a7fcbcae401d 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -8,31 +8,6 @@ import { ensureCacheDir } from '../../../util/fs'; import { regEx } from '../../../util/regex'; import type { UpdateArtifactsConfig } from '../types'; -// TODO(not7cd): rename to getPipToolsVersionConstraint, as constraints have their meaning in pip -export function getPipToolsConstraint(config: UpdateArtifactsConfig): string { - const { constraints = {} } = config; - const { pipTools } = constraints; - - if (is.string(pipTools)) { - logger.debug('Using pipTools constraint from config'); - return pipTools; - } - - return ''; -} -export function getPythonConstraint( - config: UpdateArtifactsConfig, -): string | undefined | null { - const { constraints = {} } = config; - const { python } = constraints; - - if (python) { - logger.debug('Using python constraint from config'); - return python; - } - - return undefined; -} export async function getExecOptions( config: UpdateArtifactsConfig, inputFileName: string, @@ -58,6 +33,31 @@ export async function getExecOptions( }; return execOptions; } +// TODO(not7cd): rename to getPipToolsVersionConstraint, as constraints have their meaning in pip +export function getPipToolsConstraint(config: UpdateArtifactsConfig): string { + const { constraints = {} } = config; + const { pipTools } = constraints; + + if (is.string(pipTools)) { + logger.debug('Using pipTools constraint from config'); + return pipTools; + } + + return ''; +} +export function getPythonConstraint( + config: UpdateArtifactsConfig, +): string | undefined | null { + const { constraints = {} } = config; + const { python } = constraints; + + if (python) { + logger.debug('Using python constraint from config'); + return python; + } + + return undefined; +} export const constraintLineRegex = regEx( /^(#.*?\r?\n)+# {4}(?\S*)(? .*?)?\r?\n/, From 55d48a27eee22695c171fdb10a5e7f9bd0d4ff21 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Mon, 5 Feb 2024 18:14:44 +0100 Subject: [PATCH 43/85] Move PipCompileArgs --- lib/modules/manager/pip-compile/common.ts | 11 +---------- lib/modules/manager/pip-compile/types.ts | 9 +++++++++ 2 files changed, 10 insertions(+), 10 deletions(-) create mode 100644 lib/modules/manager/pip-compile/types.ts diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index c9a7fcbcae401d..a7fa970d92d6d2 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -7,6 +7,7 @@ import type { ExecOptions } from '../../../util/exec/types'; import { ensureCacheDir } from '../../../util/fs'; import { regEx } from '../../../util/regex'; import type { UpdateArtifactsConfig } from '../types'; +import type { PipCompileArgs } from './types'; export async function getExecOptions( config: UpdateArtifactsConfig, @@ -90,16 +91,6 @@ dummyPipCompile .option('--extra-index-url ') .allowUnknownOption(); -interface PipCompileArgs { - command: string; - isCustomCommand: boolean; - outputFile?: string; - extra?: string[]; - constraint?: string[]; - sourceFiles: string[]; // positional arguments - argv: string[]; // all arguments as a list -} - // TODO(not7cd): test on all correct headers, even with CUSTOM_COMPILE_COMMAND export function extractHeaderCommand( content: string, diff --git a/lib/modules/manager/pip-compile/types.ts b/lib/modules/manager/pip-compile/types.ts new file mode 100644 index 00000000000000..195aba307d03e3 --- /dev/null +++ b/lib/modules/manager/pip-compile/types.ts @@ -0,0 +1,9 @@ +export interface PipCompileArgs { + command: string; + isCustomCommand: boolean; + outputFile?: string; + extra?: string[]; + constraint?: string[]; + sourceFiles: string[]; // positional arguments + argv: string[]; // all arguments as a list +} From c547ace1cc9c34a1a9e9a0bd8492e6e33ab6212b Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Mon, 5 Feb 2024 18:16:50 +0100 Subject: [PATCH 44/85] Move again :) --- lib/modules/manager/pip-compile/common.ts | 24 +++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index a7fa970d92d6d2..1848e3c417840a 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -34,18 +34,6 @@ export async function getExecOptions( }; return execOptions; } -// TODO(not7cd): rename to getPipToolsVersionConstraint, as constraints have their meaning in pip -export function getPipToolsConstraint(config: UpdateArtifactsConfig): string { - const { constraints = {} } = config; - const { pipTools } = constraints; - - if (is.string(pipTools)) { - logger.debug('Using pipTools constraint from config'); - return pipTools; - } - - return ''; -} export function getPythonConstraint( config: UpdateArtifactsConfig, ): string | undefined | null { @@ -59,6 +47,18 @@ export function getPythonConstraint( return undefined; } +// TODO(not7cd): rename to getPipToolsVersionConstraint, as constraints have their meaning in pip +export function getPipToolsConstraint(config: UpdateArtifactsConfig): string { + const { constraints = {} } = config; + const { pipTools } = constraints; + + if (is.string(pipTools)) { + logger.debug('Using pipTools constraint from config'); + return pipTools; + } + + return ''; +} export const constraintLineRegex = regEx( /^(#.*?\r?\n)+# {4}(?\S*)(? .*?)?\r?\n/, From f63cf8e747fea2a1b0cb6774fe3c7d9d7d8bf588 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Mon, 5 Feb 2024 18:21:11 +0100 Subject: [PATCH 45/85] Remove unused --- .../manager/pip-compile/common.spec.ts | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/lib/modules/manager/pip-compile/common.spec.ts b/lib/modules/manager/pip-compile/common.spec.ts index 2527998e888fca..12acdca5264947 100644 --- a/lib/modules/manager/pip-compile/common.spec.ts +++ b/lib/modules/manager/pip-compile/common.spec.ts @@ -1,18 +1,5 @@ -import { mockDeep } from 'jest-mock-extended'; -import { join } from 'upath'; -import { envMock } from '../../../../test/exec-util'; -import { env } from '../../../../test/util'; -import { GlobalConfig } from '../../../config/global'; -import type { RepoGlobalConfig } from '../../../config/types'; -import * as docker from '../../../util/exec/docker'; import { allowedPipOptions, extractHeaderCommand } from './common'; -jest.mock('../../../util/exec/env'); -jest.mock('../../../util/fs'); -jest.mock('../../../util/git'); -jest.mock('../../../util/host-rules', () => mockDeep()); -jest.mock('../../../util/http'); - function getCommandInHeader(command: string) { return `# # This file is autogenerated by pip-compile with Python 3.11 @@ -23,17 +10,7 @@ function getCommandInHeader(command: string) { `; } -const adminConfig: RepoGlobalConfig = { - // `join` fixes Windows CI - localDir: join('/tmp/github/some/repo'), - cacheDir: join('/tmp/renovate/cache'), - containerbaseDir: join('/tmp/renovate/cache/containerbase'), -}; - -process.env.CONTAINERBASE = 'true'; - describe('modules/manager/pip-compile/common', () => { - describe('extractHeaderCommand()', () => { it.each([ '-v', From e44eb4fbcdf699888c3f1edbfc2d690e2bcbcfc7 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Mon, 5 Feb 2024 18:25:24 +0100 Subject: [PATCH 46/85] Import quote --- lib/modules/manager/pip-compile/artifacts.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index d04fd1cdc06854..6af2c05aa70255 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -1,3 +1,4 @@ +import { quote } from 'shlex'; import { TEMPORARY_ERROR } from '../../../constants/error-messages'; import { logger } from '../../../logger'; import { exec } from '../../../util/exec'; From 8c65832dfcdc1dbbe7acae1199c95e3cca7683ac Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Tue, 6 Feb 2024 11:07:53 +0100 Subject: [PATCH 47/85] Update lib/modules/manager/pip-compile/extract.spec.ts Co-authored-by: Michael Kriese --- lib/modules/manager/pip-compile/extract.spec.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/modules/manager/pip-compile/extract.spec.ts b/lib/modules/manager/pip-compile/extract.spec.ts index 57cdbcace6fd55..fe63c0e1e401d3 100644 --- a/lib/modules/manager/pip-compile/extract.spec.ts +++ b/lib/modules/manager/pip-compile/extract.spec.ts @@ -16,8 +16,6 @@ ${deps.join('\n')}`; } describe('modules/manager/pip-compile/extract', () => { - beforeEach(() => {}); - describe('extractPackageFile()', () => { it('returns object for requirements.in', () => { expect( From 4d96cd61af21cc9d505945079c42e7c931984d58 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Tue, 6 Feb 2024 11:28:45 +0100 Subject: [PATCH 48/85] Review suggestions --- lib/modules/manager/pip-compile/common.ts | 7 +++-- .../manager/pip-compile/extract.spec.ts | 28 +++++++++---------- lib/modules/manager/pip-compile/extract.ts | 12 +++++--- lib/modules/manager/pip-compile/types.ts | 7 +++++ 4 files changed, 33 insertions(+), 21 deletions(-) diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 1848e3c417840a..7d395235130cb4 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -130,11 +130,12 @@ export function extractHeaderCommand( } } - // Commander.parse is expecting argv[0] to be process.execPath, pass empty string as first value - const parsedCommand = dummyPipCompile.parse(['', ...argv]); + const parsedCommand = dummyPipCompile.parse(argv, { from: 'user' }); const options = parsedCommand.opts(); // workaround, not sure how Commander returns named arguments - const sourceFiles = parsedCommand.args.filter((arg) => !arg.startsWith('-')); + const sourceFiles = parsedCommand.args.filter( + (arg) => !arg.startsWith('-') && arg !== command, + ); logger.trace( { argv, diff --git a/lib/modules/manager/pip-compile/extract.spec.ts b/lib/modules/manager/pip-compile/extract.spec.ts index fe63c0e1e401d3..d6c93b65a1ad80 100644 --- a/lib/modules/manager/pip-compile/extract.spec.ts +++ b/lib/modules/manager/pip-compile/extract.spec.ts @@ -41,7 +41,7 @@ describe('modules/manager/pip-compile/extract', () => { }); describe('extractAllPackageFiles()', () => { - it('support package file with multiple lock files', () => { + it('support package file with multiple lock files', async () => { fs.readLocalFile.mockResolvedValueOnce( getSimpleRequirementsFile( 'pip-compile --output-file=requirements1.txt requirements.in', @@ -68,13 +68,13 @@ describe('modules/manager/pip-compile/extract', () => { 'requirements2.txt', 'requirements3.txt', ]; - return extractAllPackageFiles({}, lockFiles).then((packageFiles) => { - expect(packageFiles).not.toBeNull(); - return expect(packageFiles[0]).toHaveProperty('lockFiles', lockFiles); - }); + const packageFiles = await extractAllPackageFiles({}, lockFiles); + expect(packageFiles).toBeDefined(); + expect(packageFiles).not.toBeNull(); + expect(packageFiles!.pop()).toHaveProperty('lockFiles', lockFiles); }); - it('no lock files in returned package files', () => { + it('no lock files in returned package files', async () => { fs.readLocalFile.mockResolvedValueOnce( getSimpleRequirementsFile('pip-compile --output-file=foo.txt foo.in', [ 'foo==1.0.1', @@ -90,15 +90,16 @@ describe('modules/manager/pip-compile/extract', () => { fs.readLocalFile.mockResolvedValueOnce('bar>=1.0.0'); const lockFiles = ['foo.txt', 'bar.txt']; - return extractAllPackageFiles({}, lockFiles).then((packageFiles) => { - return packageFiles.forEach((packageFile) => { - expect(packageFile).not.toHaveProperty('packageFile', 'foo.txt'); - }); + const packageFiles = await extractAllPackageFiles({}, lockFiles); + expect(packageFiles).toBeDefined(); + expect(packageFiles).not.toBeNull(); + packageFiles!.forEach((packageFile) => { + expect(packageFile).not.toHaveProperty('packageFile', 'foo.txt'); }); }); }); - it('return nothing for malformed files', () => { + it('return null for malformed files', async () => { fs.readLocalFile.mockResolvedValueOnce(''); fs.readLocalFile.mockResolvedValueOnce( Fixtures.get('requirementsNoHeaders.txt'), @@ -113,8 +114,7 @@ describe('modules/manager/pip-compile/extract', () => { fs.readLocalFile.mockResolvedValueOnce(''); const lockFiles = ['empty.txt', 'noHeader.txt', 'badSource.txt']; - return extractAllPackageFiles({}, lockFiles).then((packageFiles) => { - return expect(packageFiles).toBeEmptyArray(); - }); + const packageFiles = await extractAllPackageFiles({}, lockFiles); + expect(packageFiles).toBeNull(); }); }); diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index a6cb2c8fa7fe9d..927452a34e04e0 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -6,8 +6,9 @@ import { extractPackageFile as extractRequirementsFile } from '../pip_requiremen // import { extractPackageFile as extractSetupCfgFile } from '../setup-cfg'; import type { ExtractConfig, PackageFile, PackageFileContent } from '../types'; import { extractHeaderCommand } from './common'; +import type { PipCompileArgs, SupportedManagers } from './types'; -function matchManager(filename: string): string { +function matchManager(filename: string): SupportedManagers | 'unknown' { if (filename.endsWith('setup.py')) { return 'pip_setup'; } @@ -58,7 +59,7 @@ export function extractPackageFile( export async function extractAllPackageFiles( config: ExtractConfig, fileMatches: string[], -): Promise { +): Promise { logger.trace('pip-compile.extractAllPackageFiles()'); const packageFiles = new Map(); for (const fileMatch of fileMatches) { @@ -67,7 +68,7 @@ export async function extractAllPackageFiles( logger.debug(`pip-compile: no content found for fileMatch ${fileMatch}`); continue; } - let pipCompileArgs; + let pipCompileArgs: PipCompileArgs; try { pipCompileArgs = extractHeaderCommand(fileContent, fileMatch); } catch (error) { @@ -92,7 +93,7 @@ export async function extractAllPackageFiles( logger.debug( `pip-compile: ${packageFile} used in multiple output files`, ); - packageFiles.get(packageFile)?.lockFiles?.push(fileMatch); + packageFiles.get(packageFile)!.lockFiles!.push(fileMatch); continue; } const content = await readLocalFile(packageFile, 'utf8'); @@ -117,5 +118,8 @@ export async function extractAllPackageFiles( } } // TODO(not7cd): sort by requirement layering (-r -c within .in files) + if (packageFiles.size === 0) { + return null; + } return Array.from(packageFiles.values()); } diff --git a/lib/modules/manager/pip-compile/types.ts b/lib/modules/manager/pip-compile/types.ts index 195aba307d03e3..f00e7e6815c95f 100644 --- a/lib/modules/manager/pip-compile/types.ts +++ b/lib/modules/manager/pip-compile/types.ts @@ -1,3 +1,10 @@ +// managers supported by pip-tools Python package +export type SupportedManagers = + | 'pip_requirements' + | 'pip_setup' + | 'setup-cfg' + | 'pep621'; + export interface PipCompileArgs { command: string; isCustomCommand: boolean; From c5e2482f91ad693717ee00f40a6c84affc47ff8f Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Tue, 6 Feb 2024 17:21:27 +0100 Subject: [PATCH 49/85] Remove resolver support --- .../manager/pip-compile/artifacts.spec.ts | 25 +------------------ lib/modules/manager/pip-compile/artifacts.ts | 14 ----------- 2 files changed, 1 insertion(+), 38 deletions(-) diff --git a/lib/modules/manager/pip-compile/artifacts.spec.ts b/lib/modules/manager/pip-compile/artifacts.spec.ts index 25f9afaba056e7..df7de2983f3f7a 100644 --- a/lib/modules/manager/pip-compile/artifacts.spec.ts +++ b/lib/modules/manager/pip-compile/artifacts.spec.ts @@ -10,7 +10,7 @@ import * as docker from '../../../util/exec/docker'; import type { StatusResult } from '../../../util/git/types'; import * as _datasource from '../../datasource'; import type { UpdateArtifactsConfig } from '../types'; -import { constructPipCompileCmd, extractResolver } from './artifacts'; +import { constructPipCompileCmd } from './artifacts'; import { updateArtifacts } from '.'; const datasource = mocked(_datasource); @@ -374,27 +374,4 @@ describe('modules/manager/pip-compile/artifacts', () => { // ); // }); }); - - describe('extractResolver()', () => { - it.each([ - ['--resolver=backtracking', 'backtracking'], - ['--resolver=legacy', 'legacy'], - ])( - 'returns expected value for supported %s resolver', - (argument: string, expected: string) => { - expect(extractResolver(argument)).toBe(expected); - }, - ); - - it.each(['--resolver=foo', '--resolver='])( - 'returns null for unsupported %s resolver', - (argument: string) => { - expect(extractResolver(argument)).toBeNull(); - expect(logger.warn).toHaveBeenCalledWith( - { argument }, - 'pip-compile was previously executed with an unexpected `--resolver` value', - ); - }, - ); - }); }); diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index 6af2c05aa70255..3a532207ce3132 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -92,17 +92,3 @@ export async function updateArtifacts({ logger.debug('Returning updated pip-compile result'); return result; } - -// TODO(not7cd): remove, legacy resolver is deprecated and will be removed -export function extractResolver(argument: string): string | null { - const value = argument.replace('--resolver=', ''); - if (['backtracking', 'legacy'].includes(value)) { - return value; - } - - logger.warn( - { argument }, - 'pip-compile was previously executed with an unexpected `--resolver` value', - ); - return null; -} From 75763e5ca6cb3c334a50ea410cfc0bcdda9591e8 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Tue, 6 Feb 2024 19:07:35 +0100 Subject: [PATCH 50/85] test to it Co-authored-by: Michael Kriese --- lib/modules/manager/pip-compile/common.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/modules/manager/pip-compile/common.spec.ts b/lib/modules/manager/pip-compile/common.spec.ts index 12acdca5264947..e4db89e7e290f2 100644 --- a/lib/modules/manager/pip-compile/common.spec.ts +++ b/lib/modules/manager/pip-compile/common.spec.ts @@ -63,7 +63,7 @@ describe('modules/manager/pip-compile/common', () => { }, ); - test('error when no source files passed as arguments', () => { + it('error when no source files passed as arguments', () => { expect(() => extractHeaderCommand( getCommandInHeader(`pip-compile --extra=color`), @@ -72,7 +72,7 @@ describe('modules/manager/pip-compile/common', () => { ).toThrow(/source/); }); - test('returned sourceFiles returns all source files', () => { + it('returned sourceFiles returns all source files', () => { const exampleSourceFiles = [ 'requirements.in', 'reqs/testing.in', From 9e48249bc71dd17905d79614c3bb84abd1a7a0bf Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Wed, 7 Feb 2024 10:51:18 +0100 Subject: [PATCH 51/85] Remove type Co-authored-by: Michael Kriese --- lib/modules/manager/pip-compile/common.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 7d395235130cb4..643b0b245495cd 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -105,7 +105,7 @@ export function extractHeaderCommand( `pip-compile: found header in ${fileName}: \n${compileCommand[0]}`, ); const command = compileCommand.groups.command; - const argv: string[] = [command]; + const argv = [command]; const isCustomCommand = command !== 'pip-compile'; if (isCustomCommand) { logger.debug( From 828b7fd2b777b0a404562db86f9846286c91eb00 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Wed, 7 Feb 2024 12:35:20 +0100 Subject: [PATCH 52/85] Don't parse with commander.js --- lib/modules/manager/pip-compile/common.ts | 73 ++++++++++++----------- lib/modules/manager/pip-compile/types.ts | 2 + 2 files changed, 40 insertions(+), 35 deletions(-) diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 643b0b245495cd..5e3a1e08138be4 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -2,6 +2,7 @@ import is from '@sindresorhus/is'; import { Command } from 'commander'; import { quote, split } from 'shlex'; import upath from 'upath'; +import { camelCase } from '../../../../tools/utils'; import { logger } from '../../../logger'; import type { ExecOptions } from '../../../util/exec/types'; import { ensureCacheDir } from '../../../util/fs'; @@ -79,18 +80,10 @@ export const allowedPipOptions = [ '--generate-hashes', '--no-emit-index-url', // TODO: handle this!!! '--strip-extras', + '--index-url', ...optionsWithArguments, ]; -// as commander.js is already used, we will reuse it's argument parsing capability -const dummyPipCompile = new Command(); -dummyPipCompile - .argument('[sourceFile...]') // optional, so extractHeaderCommand can throw an explicit error - .option('--output-file ') - .option('--extra ') - .option('--extra-index-url ') - .allowUnknownOption(); - // TODO(not7cd): test on all correct headers, even with CUSTOM_COMPILE_COMMAND export function extractHeaderCommand( content: string, @@ -118,9 +111,19 @@ export function extractHeaderCommand( logger.debug( `pip-compile: extracted command from header: ${JSON.stringify(argv)}`, ); - for (const arg of argv) { + + const result: PipCompileArgs = { + argv, + command, + isCustomCommand, + outputFile: '', + sourceFiles: [], + }; + // const options: Record = {}; + for (const arg of argv.slice(1)) { // TODO(not7cd): check for "--option -- argument" case if (!arg.startsWith('-')) { + result.sourceFiles.push(arg); continue; } throwForDisallowedOption(arg); @@ -128,43 +131,49 @@ export function extractHeaderCommand( if (strict) { throwForUnknownOption(arg); } + + if (arg.includes('=')) { + const [option, value] = arg.split('='); + if (option === '--extra') { + result.extra = result.extra ?? []; + result.extra.push(value); + } else if (option === '--extra-index-url') { + result.extraIndexUrl = result.extraIndexUrl ?? []; + result.extraIndexUrl.push(value); + // TODO: add to secrets + } else if (option === '--output-file') { + result.outputFile = value; + } else if (option === '--index-url') { + result.indexUrl = value; + // TODO: add to secrets + } + continue; + } } - const parsedCommand = dummyPipCompile.parse(argv, { from: 'user' }); - const options = parsedCommand.opts(); - // workaround, not sure how Commander returns named arguments - const sourceFiles = parsedCommand.args.filter( - (arg) => !arg.startsWith('-') && arg !== command, - ); logger.trace( { - argv, - options, - sourceFiles, - isCustomCommand, + ...result, }, 'Parsed pip-compile command from header', ); - if (sourceFiles.length === 0) { + if (result.sourceFiles.length === 0) { throw new Error( 'No source files detected in command, pass at least one package file explicitly', ); } - let outputFile = ''; - if (options.outputFile) { + if (result.outputFile) { // TODO(not7cd): This file path can be relative like `reqs/main.txt` const file = upath.parse(fileName).base; - if (options.outputFile === file) { - outputFile = options.outputFile; - } else { + if (result.outputFile !== file) { // we don't trust the user-supplied output-file argument; // TODO(not7cd): allow relative paths logger.warn( - { outputFile: options.outputFile, actualPath: file }, + { outputFile: result.outputFile, actualPath: file }, 'pip-compile was previously executed with an unexpected `--output-file` filename', ); // TODO(not7cd): this shouldn't be changed in extract function - outputFile = file; + result.outputFile = file; argv.forEach((item, i) => { if (item.startsWith('--output-file=')) { argv[i] = `--output-file=${quote(file)}`; @@ -174,13 +183,7 @@ export function extractHeaderCommand( } else { logger.debug(`pip-compile: implicit output file (${fileName})`); } - return { - argv, - command, - isCustomCommand, - outputFile, - sourceFiles, - }; + return result; } function throwForDisallowedOption(arg: string): void { diff --git a/lib/modules/manager/pip-compile/types.ts b/lib/modules/manager/pip-compile/types.ts index f00e7e6815c95f..12dd3ace59488b 100644 --- a/lib/modules/manager/pip-compile/types.ts +++ b/lib/modules/manager/pip-compile/types.ts @@ -8,6 +8,8 @@ export type SupportedManagers = export interface PipCompileArgs { command: string; isCustomCommand: boolean; + indexUrl?: string; + extraIndexUrl?: string[]; outputFile?: string; extra?: string[]; constraint?: string[]; From 04307c88d13c884f78c0837d3c85c56d5dd79688 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Wed, 7 Feb 2024 12:51:47 +0100 Subject: [PATCH 53/85] Explicit test --- lib/modules/manager/pip-compile/common.ts | 6 ++---- lib/modules/manager/pip-compile/extract.spec.ts | 14 +++++++------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 5e3a1e08138be4..0d5ba0db401a23 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -1,8 +1,6 @@ import is from '@sindresorhus/is'; -import { Command } from 'commander'; import { quote, split } from 'shlex'; import upath from 'upath'; -import { camelCase } from '../../../../tools/utils'; import { logger } from '../../../logger'; import type { ExecOptions } from '../../../util/exec/types'; import { ensureCacheDir } from '../../../util/fs'; @@ -140,12 +138,12 @@ export function extractHeaderCommand( } else if (option === '--extra-index-url') { result.extraIndexUrl = result.extraIndexUrl ?? []; result.extraIndexUrl.push(value); - // TODO: add to secrets + // TODO: add to secrets? next PR } else if (option === '--output-file') { result.outputFile = value; } else if (option === '--index-url') { result.indexUrl = value; - // TODO: add to secrets + // TODO: add to secrets? next PR } continue; } diff --git a/lib/modules/manager/pip-compile/extract.spec.ts b/lib/modules/manager/pip-compile/extract.spec.ts index d6c93b65a1ad80..fd725ef2802d28 100644 --- a/lib/modules/manager/pip-compile/extract.spec.ts +++ b/lib/modules/manager/pip-compile/extract.spec.ts @@ -18,13 +18,13 @@ ${deps.join('\n')}`; describe('modules/manager/pip-compile/extract', () => { describe('extractPackageFile()', () => { it('returns object for requirements.in', () => { - expect( - extractPackageFile( - Fixtures.get('requirementsWithHashes.txt'), - 'requirements.in', - {}, - ), - ).toBeObject(); + const packageFile = extractPackageFile( + Fixtures.get('requirementsWithHashes.txt'), + 'requirements.in', + {}, + ); + expect(packageFile).toHaveProperty('deps'); + expect(packageFile?.deps[0]).toHaveProperty('depName', 'attrs'); }); it.each([ From 2cd997a1295d6d48bc14dddd27e979027be87ebb Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Wed, 7 Feb 2024 12:57:56 +0100 Subject: [PATCH 54/85] Prefix all logger.debug --- lib/modules/manager/pip-compile/artifacts.ts | 6 +++--- lib/modules/manager/pip-compile/common.ts | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index 3a532207ce3132..22d28619d7ec62 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -48,7 +48,7 @@ export async function updateArtifacts({ for (const outputFileName of config.lockFiles) { const existingOutput = await readLocalFile(outputFileName, 'utf8'); if (!existingOutput) { - logger.debug('No pip-compile output file found'); + logger.debug('pip-compile: No output file found'); return null; } try { @@ -80,7 +80,7 @@ export async function updateArtifacts({ if (err.message === TEMPORARY_ERROR) { throw err; } - logger.debug({ err }, 'Failed to pip-compile'); + logger.debug({ err }, 'pip-compile: Failed to run command'); result.push({ artifactError: { lockFile: outputFileName, @@ -89,6 +89,6 @@ export async function updateArtifacts({ }); } } - logger.debug('Returning updated pip-compile result'); + logger.debug('pip-compile: Returning updated output file(s)'); return result; } diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 0d5ba0db401a23..3bb3a6ad6801e0 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -40,7 +40,7 @@ export function getPythonConstraint( const { python } = constraints; if (python) { - logger.debug('Using python constraint from config'); + logger.debug('pip-compile: Using python constraint from config'); return python; } @@ -52,7 +52,7 @@ export function getPipToolsConstraint(config: UpdateArtifactsConfig): string { const { pipTools } = constraints; if (is.string(pipTools)) { - logger.debug('Using pipTools constraint from config'); + logger.debug('pip-compile: Using pipTools constraint from config'); return pipTools; } From 9cf030a29dd79ac3a2cb554e57a5022e0972f4ef Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Wed, 7 Feb 2024 12:59:48 +0100 Subject: [PATCH 55/85] Prefix logger.warn --- lib/modules/manager/pip-compile/artifacts.ts | 2 +- lib/modules/manager/pip-compile/extract.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index 22d28619d7ec62..e71ddd6f4ca18a 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -35,7 +35,7 @@ export async function updateArtifacts({ if (!config.lockFiles) { logger.warn( { packageFileName: inputFileName }, - 'No lock files associated with a package file', + 'pip-compile: No lock files associated with a package file', ); return null; } diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index 927452a34e04e0..1ff193a9377528 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -74,7 +74,7 @@ export async function extractAllPackageFiles( } catch (error) { logger.warn( { fileMatch, error }, - 'Failed to parse pip-compile command from header', + 'pip-compile: Failed to extract and parse command in output file header', ); continue; } From fed1c24a9c79f8d9e9d2673fb8c18025c7b40d99 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Wed, 7 Feb 2024 14:38:42 +0100 Subject: [PATCH 56/85] Add dependency flowchart for debugging --- lib/modules/manager/pip-compile/extract.ts | 39 ++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index 1ff193a9377528..bce0b9f0b9e7bf 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -56,11 +56,40 @@ export function extractPackageFile( } } +function generateMermaidFlowchart( + packageFiles: PackageFile[], + lockFileArgs: Map, + sourceLockFiles: Array<{ sourceFile: string; lockFile: string }>, +): string { + const lockFiles = []; + for (const [lockFile, pipCompileArgs] of lockFileArgs.entries()) { + const extraArgs = pipCompileArgs.extra + ?.map((v) => '--extra=' + v) + .join('\n'); + lockFiles.push( + ` ${lockFile}[[${lockFile}${extraArgs ? '\n' + extraArgs : ''}]]`, + ); + } + + const edges = packageFiles.flatMap((packageFile) => { + return packageFile.lockFiles!.map((lockFile) => { + return ` ${packageFile.packageFile} --> ${lockFile}`; + }); + }); + const lockEdges = sourceLockFiles.map(({ sourceFile, lockFile }) => { + return ` ${sourceFile} --> ${lockFile}`; + }); + return `graph TD\n${lockFiles.join('\n')}\n${edges.join('\n')}\n${lockEdges.join('\n')}`; +} + export async function extractAllPackageFiles( config: ExtractConfig, fileMatches: string[], ): Promise { logger.trace('pip-compile.extractAllPackageFiles()'); + const lockFileArgs = new Map(); + const sourceLockFiles = []; + // for debugging only ^^^ (for now) const packageFiles = new Map(); for (const fileMatch of fileMatches) { const fileContent = await readLocalFile(fileMatch, 'utf8'); @@ -71,6 +100,7 @@ export async function extractAllPackageFiles( let pipCompileArgs: PipCompileArgs; try { pipCompileArgs = extractHeaderCommand(fileContent, fileMatch); + lockFileArgs.set(fileMatch, pipCompileArgs); } catch (error) { logger.warn( { fileMatch, error }, @@ -87,6 +117,7 @@ export async function extractAllPackageFiles( { sourceFile: packageFile, lockFile: fileMatch }, 'pip-compile: lock file acts as source file for another lock file', ); + sourceLockFiles.push({ sourceFile: packageFile, lockFile: fileMatch }); continue; } if (packageFiles.has(packageFile)) { @@ -121,5 +152,13 @@ export async function extractAllPackageFiles( if (packageFiles.size === 0) { return null; } + logger.debug( + 'pip-compile: dependency flowchart:\n' + + generateMermaidFlowchart( + Array.from(packageFiles.values()), + lockFileArgs, + sourceLockFiles, + ), + ); return Array.from(packageFiles.values()); } From e380d704458433d6e242ba7196d074a90b147b61 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Wed, 7 Feb 2024 14:45:07 +0100 Subject: [PATCH 57/85] Fix test --- lib/modules/manager/pip-compile/artifacts.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/manager/pip-compile/artifacts.spec.ts b/lib/modules/manager/pip-compile/artifacts.spec.ts index df7de2983f3f7a..685e65b89f85d8 100644 --- a/lib/modules/manager/pip-compile/artifacts.spec.ts +++ b/lib/modules/manager/pip-compile/artifacts.spec.ts @@ -108,7 +108,7 @@ describe('modules/manager/pip-compile/artifacts', () => { ).toBeNull(); expect(logger.warn).toHaveBeenCalledWith( { packageFileName: 'requirements.in' }, - 'No lock files associated with a package file', + 'pip-compile: No lock files associated with a package file', ); }); From 2eaf8923cfe83656c15d73e2423df8ab48a646c9 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Wed, 7 Feb 2024 14:54:16 +0100 Subject: [PATCH 58/85] Refactor dependency flowchart --- lib/modules/manager/pip-compile/extract.ts | 31 +++++++++------------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index bce0b9f0b9e7bf..28594ee5bf68f0 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -57,9 +57,8 @@ export function extractPackageFile( } function generateMermaidFlowchart( - packageFiles: PackageFile[], + depsBetweenFiles: Array<{ sourceFile: string; outputFile: string }>, lockFileArgs: Map, - sourceLockFiles: Array<{ sourceFile: string; lockFile: string }>, ): string { const lockFiles = []; for (const [lockFile, pipCompileArgs] of lockFileArgs.entries()) { @@ -70,16 +69,10 @@ function generateMermaidFlowchart( ` ${lockFile}[[${lockFile}${extraArgs ? '\n' + extraArgs : ''}]]`, ); } - - const edges = packageFiles.flatMap((packageFile) => { - return packageFile.lockFiles!.map((lockFile) => { - return ` ${packageFile.packageFile} --> ${lockFile}`; - }); - }); - const lockEdges = sourceLockFiles.map(({ sourceFile, lockFile }) => { - return ` ${sourceFile} --> ${lockFile}`; + const edges = depsBetweenFiles.map(({ sourceFile, outputFile }) => { + return ` ${sourceFile} --> ${outputFile}`; }); - return `graph TD\n${lockFiles.join('\n')}\n${edges.join('\n')}\n${lockEdges.join('\n')}`; + return `graph TD\n${lockFiles.join('\n')}\n${edges.join('\n')}`; } export async function extractAllPackageFiles( @@ -88,7 +81,10 @@ export async function extractAllPackageFiles( ): Promise { logger.trace('pip-compile.extractAllPackageFiles()'); const lockFileArgs = new Map(); - const sourceLockFiles = []; + const depsBetweenFiles = new Array<{ + sourceFile: string; + outputFile: string; + }>(); // for debugging only ^^^ (for now) const packageFiles = new Map(); for (const fileMatch of fileMatches) { @@ -111,13 +107,16 @@ export async function extractAllPackageFiles( // TODO(not7cd): handle locked deps // const lockedDeps = extractRequirementsFile(content); for (const packageFile of pipCompileArgs.sourceFiles) { + depsBetweenFiles.push({ + sourceFile: packageFile, + outputFile: fileMatch, + }); if (fileMatches.includes(packageFile)) { // TODO(not7cd): do something about it logger.warn( { sourceFile: packageFile, lockFile: fileMatch }, 'pip-compile: lock file acts as source file for another lock file', ); - sourceLockFiles.push({ sourceFile: packageFile, lockFile: fileMatch }); continue; } if (packageFiles.has(packageFile)) { @@ -154,11 +153,7 @@ export async function extractAllPackageFiles( } logger.debug( 'pip-compile: dependency flowchart:\n' + - generateMermaidFlowchart( - Array.from(packageFiles.values()), - lockFileArgs, - sourceLockFiles, - ), + generateMermaidFlowchart(depsBetweenFiles, lockFileArgs), ); return Array.from(packageFiles.values()); } From 1fd5ce0caebfde6085f9ae5a749edba792212eb3 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Wed, 7 Feb 2024 14:59:51 +0100 Subject: [PATCH 59/85] Disable unused and untested part flowchart --- lib/modules/manager/pip-compile/extract.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index 28594ee5bf68f0..4bb0bfbb6ba594 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -61,13 +61,12 @@ function generateMermaidFlowchart( lockFileArgs: Map, ): string { const lockFiles = []; - for (const [lockFile, pipCompileArgs] of lockFileArgs.entries()) { - const extraArgs = pipCompileArgs.extra - ?.map((v) => '--extra=' + v) - .join('\n'); - lockFiles.push( - ` ${lockFile}[[${lockFile}${extraArgs ? '\n' + extraArgs : ''}]]`, - ); + for (const lockFile of lockFileArgs.keys()) { + // TODO: add extra args to the lock file ${extraArgs ? '\n' + extraArgs : ''} + // const extraArgs = pipCompileArgs.extra + // ?.map((v) => '--extra=' + v) + // .join('\n'); + lockFiles.push(` ${lockFile}[[${lockFile}]]`); } const edges = depsBetweenFiles.map(({ sourceFile, outputFile }) => { return ` ${sourceFile} --> ${outputFile}`; From 394b68590a79555f03076e57a8ac08cc75ae9c94 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Wed, 7 Feb 2024 17:51:12 +0100 Subject: [PATCH 60/85] Extract constraints from header --- lib/modules/manager/pip-compile/common.ts | 4 ++++ lib/modules/manager/pip-compile/extract.ts | 25 +++++++++++++++------- lib/modules/manager/pip-compile/types.ts | 8 ++++++- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 3bb3a6ad6801e0..2df347763fd4d6 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -71,6 +71,7 @@ export const optionsWithArguments = [ '--extra', '--extra-index-url', '--resolver', + '--constraint', ]; export const allowedPipOptions = [ '-v', @@ -139,6 +140,9 @@ export function extractHeaderCommand( result.extraIndexUrl = result.extraIndexUrl ?? []; result.extraIndexUrl.push(value); // TODO: add to secrets? next PR + } else if (option === '--constraint') { + result.constraints = result.constraints ?? []; + result.constraints.push(value); } else if (option === '--output-file') { result.outputFile = value; } else if (option === '--index-url') { diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index 4bb0bfbb6ba594..f370a82b2b88b0 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -6,7 +6,11 @@ import { extractPackageFile as extractRequirementsFile } from '../pip_requiremen // import { extractPackageFile as extractSetupCfgFile } from '../setup-cfg'; import type { ExtractConfig, PackageFile, PackageFileContent } from '../types'; import { extractHeaderCommand } from './common'; -import type { PipCompileArgs, SupportedManagers } from './types'; +import type { + DependencyBetweenFiles, + PipCompileArgs, + SupportedManagers, +} from './types'; function matchManager(filename: string): SupportedManagers | 'unknown' { if (filename.endsWith('setup.py')) { @@ -57,7 +61,7 @@ export function extractPackageFile( } function generateMermaidFlowchart( - depsBetweenFiles: Array<{ sourceFile: string; outputFile: string }>, + depsBetweenFiles: DependencyBetweenFiles[], lockFileArgs: Map, ): string { const lockFiles = []; @@ -68,8 +72,8 @@ function generateMermaidFlowchart( // .join('\n'); lockFiles.push(` ${lockFile}[[${lockFile}]]`); } - const edges = depsBetweenFiles.map(({ sourceFile, outputFile }) => { - return ` ${sourceFile} --> ${outputFile}`; + const edges = depsBetweenFiles.map(({ sourceFile, outputFile, type }) => { + return ` ${sourceFile} -${type === 'constraint' ? '.' : ''}-> ${outputFile}`; }); return `graph TD\n${lockFiles.join('\n')}\n${edges.join('\n')}`; } @@ -80,10 +84,7 @@ export async function extractAllPackageFiles( ): Promise { logger.trace('pip-compile.extractAllPackageFiles()'); const lockFileArgs = new Map(); - const depsBetweenFiles = new Array<{ - sourceFile: string; - outputFile: string; - }>(); + const depsBetweenFiles = new Array(); // for debugging only ^^^ (for now) const packageFiles = new Map(); for (const fileMatch of fileMatches) { @@ -103,12 +104,20 @@ export async function extractAllPackageFiles( ); continue; } + for (const constraint in pipCompileArgs.constraints) { + depsBetweenFiles.push({ + sourceFile: constraint, + outputFile: fileMatch, + type: 'constraint', + }); + } // TODO(not7cd): handle locked deps // const lockedDeps = extractRequirementsFile(content); for (const packageFile of pipCompileArgs.sourceFiles) { depsBetweenFiles.push({ sourceFile: packageFile, outputFile: fileMatch, + type: 'requirement', }); if (fileMatches.includes(packageFile)) { // TODO(not7cd): do something about it diff --git a/lib/modules/manager/pip-compile/types.ts b/lib/modules/manager/pip-compile/types.ts index 12dd3ace59488b..69d8dc57fe3b04 100644 --- a/lib/modules/manager/pip-compile/types.ts +++ b/lib/modules/manager/pip-compile/types.ts @@ -12,7 +12,13 @@ export interface PipCompileArgs { extraIndexUrl?: string[]; outputFile?: string; extra?: string[]; - constraint?: string[]; + constraints?: string[]; sourceFiles: string[]; // positional arguments argv: string[]; // all arguments as a list } + +export interface DependencyBetweenFiles { + sourceFile: string; + outputFile: string; + type: 'requirement' | 'constraint'; +} From 23833f44921923d2b99f14d8464f25cba747408a Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Wed, 7 Feb 2024 20:25:09 +0100 Subject: [PATCH 61/85] Disable coverage for constraints They are only parsed but not handled. For a feature request. --- lib/modules/manager/pip-compile/extract.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index f370a82b2b88b0..d4f0d1bf0addd3 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -105,6 +105,8 @@ export async function extractAllPackageFiles( continue; } for (const constraint in pipCompileArgs.constraints) { + // TODO(not7cd): handle constraints + /* istanbul ignore next */ depsBetweenFiles.push({ sourceFile: constraint, outputFile: fileMatch, From 7585f4ce53db2aff9596fcad1a53d0ee56b916a7 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Wed, 7 Feb 2024 20:26:24 +0100 Subject: [PATCH 62/85] Rename to contraintsFiles --- lib/modules/manager/pip-compile/common.ts | 4 ++-- lib/modules/manager/pip-compile/extract.ts | 2 +- lib/modules/manager/pip-compile/types.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 2df347763fd4d6..29037efc13c934 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -141,8 +141,8 @@ export function extractHeaderCommand( result.extraIndexUrl.push(value); // TODO: add to secrets? next PR } else if (option === '--constraint') { - result.constraints = result.constraints ?? []; - result.constraints.push(value); + result.constraintFiles = result.constraintFiles ?? []; + result.constraintFiles.push(value); } else if (option === '--output-file') { result.outputFile = value; } else if (option === '--index-url') { diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index d4f0d1bf0addd3..f0c6cd4d376e70 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -104,7 +104,7 @@ export async function extractAllPackageFiles( ); continue; } - for (const constraint in pipCompileArgs.constraints) { + for (const constraint in pipCompileArgs.constraintFiles) { // TODO(not7cd): handle constraints /* istanbul ignore next */ depsBetweenFiles.push({ diff --git a/lib/modules/manager/pip-compile/types.ts b/lib/modules/manager/pip-compile/types.ts index 69d8dc57fe3b04..9a2112c49d0c4e 100644 --- a/lib/modules/manager/pip-compile/types.ts +++ b/lib/modules/manager/pip-compile/types.ts @@ -12,7 +12,7 @@ export interface PipCompileArgs { extraIndexUrl?: string[]; outputFile?: string; extra?: string[]; - constraints?: string[]; + constraintFiles?: string[]; sourceFiles: string[]; // positional arguments argv: string[]; // all arguments as a list } From ff456d0b36f52634ea1ec35055347118358441ad Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Wed, 7 Feb 2024 20:27:35 +0100 Subject: [PATCH 63/85] Sort --- lib/modules/manager/pip-compile/common.ts | 4 ++-- lib/modules/manager/pip-compile/extract.ts | 2 +- lib/modules/manager/pip-compile/types.ts | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 29037efc13c934..05b0040a18e5f9 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -141,8 +141,8 @@ export function extractHeaderCommand( result.extraIndexUrl.push(value); // TODO: add to secrets? next PR } else if (option === '--constraint') { - result.constraintFiles = result.constraintFiles ?? []; - result.constraintFiles.push(value); + result.constraintsFiles = result.constraintsFiles ?? []; + result.constraintsFiles.push(value); } else if (option === '--output-file') { result.outputFile = value; } else if (option === '--index-url') { diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index f0c6cd4d376e70..60ad28a7ca89e8 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -104,7 +104,7 @@ export async function extractAllPackageFiles( ); continue; } - for (const constraint in pipCompileArgs.constraintFiles) { + for (const constraint in pipCompileArgs.constraintsFiles) { // TODO(not7cd): handle constraints /* istanbul ignore next */ depsBetweenFiles.push({ diff --git a/lib/modules/manager/pip-compile/types.ts b/lib/modules/manager/pip-compile/types.ts index 9a2112c49d0c4e..6feb336e2458c5 100644 --- a/lib/modules/manager/pip-compile/types.ts +++ b/lib/modules/manager/pip-compile/types.ts @@ -6,15 +6,15 @@ export type SupportedManagers = | 'pep621'; export interface PipCompileArgs { + argv: string[]; // all arguments as a list command: string; isCustomCommand: boolean; - indexUrl?: string; + constraintsFiles?: string[]; + extra?: string[]; extraIndexUrl?: string[]; + indexUrl?: string; outputFile?: string; - extra?: string[]; - constraintFiles?: string[]; sourceFiles: string[]; // positional arguments - argv: string[]; // all arguments as a list } export interface DependencyBetweenFiles { From 6aa80eade71c174f0fd58aa65e5761ba8f17d140 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Wed, 7 Feb 2024 20:33:10 +0100 Subject: [PATCH 64/85] Rename flowchart to graph --- lib/modules/manager/pip-compile/extract.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index 60ad28a7ca89e8..5865ee74b681b0 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -60,7 +60,7 @@ export function extractPackageFile( } } -function generateMermaidFlowchart( +function generateMermaidGraph( depsBetweenFiles: DependencyBetweenFiles[], lockFileArgs: Map, ): string { @@ -162,8 +162,8 @@ export async function extractAllPackageFiles( return null; } logger.debug( - 'pip-compile: dependency flowchart:\n' + - generateMermaidFlowchart(depsBetweenFiles, lockFileArgs), + 'pip-compile: dependency graph:\n' + + generateMermaidGraph(depsBetweenFiles, lockFileArgs), ); return Array.from(packageFiles.values()); } From b4770e2825ab4f22f520af7df437369e8d9d19c6 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Wed, 7 Feb 2024 20:37:28 +0100 Subject: [PATCH 65/85] Log header with trace Printing whole header is too verbose --- lib/modules/manager/pip-compile/common.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 05b0040a18e5f9..effce27ac824dc 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -93,7 +93,7 @@ export function extractHeaderCommand( if (compileCommand?.groups === undefined) { throw new Error(`Failed to extract command from header in ${fileName}`); } - logger.debug( + logger.trace( `pip-compile: found header in ${fileName}: \n${compileCommand[0]}`, ); const command = compileCommand.groups.command; From 8290a8c7e89694d3a5db451a2913cac4915a73bf Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Wed, 7 Feb 2024 21:17:01 +0100 Subject: [PATCH 66/85] types Co-authored-by: Michael Kriese --- lib/modules/manager/pip-compile/artifacts.ts | 2 +- lib/modules/manager/pip-compile/extract.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index e71ddd6f4ca18a..b22b0308706702 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -58,7 +58,7 @@ export async function updateArtifacts({ await deleteLocalFile(outputFileName); } const cmd = constructPipCompileCmd(existingOutput, outputFileName); - const execOptions: ExecOptions = await getExecOptions( + const execOptions = await getExecOptions( config, inputFileName, ); diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index 5865ee74b681b0..9ec25404586cf2 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -84,7 +84,7 @@ export async function extractAllPackageFiles( ): Promise { logger.trace('pip-compile.extractAllPackageFiles()'); const lockFileArgs = new Map(); - const depsBetweenFiles = new Array(); + const depsBetweenFiles: DependencyBetweenFiles[] = []; // for debugging only ^^^ (for now) const packageFiles = new Map(); for (const fileMatch of fileMatches) { From 3e473066a9456c9b93caefbd8e3c620b8dc67b3c Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Wed, 7 Feb 2024 21:18:51 +0100 Subject: [PATCH 67/85] nit --- lib/modules/manager/pip-compile/artifacts.ts | 6 +----- lib/modules/manager/pip-compile/common.ts | 4 ++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index b22b0308706702..ed7a0350df1b17 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -2,7 +2,6 @@ import { quote } from 'shlex'; import { TEMPORARY_ERROR } from '../../../constants/error-messages'; import { logger } from '../../../logger'; import { exec } from '../../../util/exec'; -import type { ExecOptions } from '../../../util/exec/types'; import { deleteLocalFile, readLocalFile, @@ -58,10 +57,7 @@ export async function updateArtifacts({ await deleteLocalFile(outputFileName); } const cmd = constructPipCompileCmd(existingOutput, outputFileName); - const execOptions = await getExecOptions( - config, - inputFileName, - ); + const execOptions = await getExecOptions(config, inputFileName); logger.trace({ cmd }, 'pip-compile command'); await exec(cmd, execOptions); const status = await getRepoStatus(); diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index effce27ac824dc..4cd4eb82c946ab 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -40,7 +40,7 @@ export function getPythonConstraint( const { python } = constraints; if (python) { - logger.debug('pip-compile: Using python constraint from config'); + logger.debug('Using python constraint from config'); return python; } @@ -52,7 +52,7 @@ export function getPipToolsConstraint(config: UpdateArtifactsConfig): string { const { pipTools } = constraints; if (is.string(pipTools)) { - logger.debug('pip-compile: Using pipTools constraint from config'); + logger.debug('Using pipTools constraint from config'); return pipTools; } From ada4937ba5922aa7121508d60da65737146d6e3f Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Thu, 8 Feb 2024 10:37:13 +0100 Subject: [PATCH 68/85] Update tests --- .../manager/pip-compile/extract.spec.ts | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/lib/modules/manager/pip-compile/extract.spec.ts b/lib/modules/manager/pip-compile/extract.spec.ts index fd725ef2802d28..1f53a84b2ab385 100644 --- a/lib/modules/manager/pip-compile/extract.spec.ts +++ b/lib/modules/manager/pip-compile/extract.spec.ts @@ -92,11 +92,31 @@ describe('modules/manager/pip-compile/extract', () => { const lockFiles = ['foo.txt', 'bar.txt']; const packageFiles = await extractAllPackageFiles({}, lockFiles); expect(packageFiles).toBeDefined(); - expect(packageFiles).not.toBeNull(); packageFiles!.forEach((packageFile) => { expect(packageFile).not.toHaveProperty('packageFile', 'foo.txt'); }); }); + + // TODO(not7cd): update when constraints are supported + it('no constraint files in returned package files', async () => { + fs.readLocalFile.mockResolvedValueOnce( + getSimpleRequirementsFile( + 'pip-compile --output-file=requirements.txt --constraint=constraints.txt requirements.in', + ['foo==1.0.1'], + ), + ); + fs.readLocalFile.mockResolvedValueOnce('foo>=1.0.0'); + + const lockFiles = ['requirements.txt']; + const packageFiles = await extractAllPackageFiles({}, lockFiles); + expect(packageFiles).toBeDefined(); + packageFiles!.forEach((packageFile) => { + expect(packageFile).not.toHaveProperty( + 'packageFile', + 'constraints.txt', + ); + }); + }); }); it('return null for malformed files', async () => { From b8a5bb8f015032e795f38d069a45d6186112deef Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Thu, 8 Feb 2024 10:57:14 +0100 Subject: [PATCH 69/85] Fix option checks --- .../manager/pip-compile/common.spec.ts | 3 ++- lib/modules/manager/pip-compile/common.ts | 27 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/lib/modules/manager/pip-compile/common.spec.ts b/lib/modules/manager/pip-compile/common.spec.ts index e4db89e7e290f2..3eb41ab5c32b02 100644 --- a/lib/modules/manager/pip-compile/common.spec.ts +++ b/lib/modules/manager/pip-compile/common.spec.ts @@ -18,6 +18,7 @@ describe('modules/manager/pip-compile/common', () => { '--resolver=backtracking', '--resolver=legacy', '--output-file=reqs.txt', + '--extra-index-url=https://pypi.org/simple', ])('returns object on correct options', (argument: string) => { expect( extractHeaderCommand( @@ -39,7 +40,7 @@ describe('modules/manager/pip-compile/common', () => { }, ); - it.each(['--foo', '-x', '--$(curl this)', '--bar=sus'])( + it.each(['--foo', '-x', '--$(curl this)', '--bar=sus', '--extra-large'])( 'errors on unknown options', (argument: string) => { expect(() => diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 4cd4eb82c946ab..300327e0f1e54f 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -189,30 +189,29 @@ export function extractHeaderCommand( } function throwForDisallowedOption(arg: string): void { - for (const disallowedPipOption of disallowedPipOptions) { - if (arg.startsWith(disallowedPipOption)) { - throw new Error( - `Option ${disallowedPipOption} not allowed for this manager`, - ); - } + if (disallowedPipOptions.includes(arg)) { + throw new Error(`Option ${arg} not allowed for this manager`); } } function throwForNoEqualSignInOptionWithArgument(arg: string): void { - for (const option of optionsWithArguments) { - if (arg === option) { - throw new Error( - `Option ${option} must have equal sign '=' separating it's argument`, - ); - } + // this won't match if there is `=` at the end of the string + if (optionsWithArguments.includes(arg)) { + throw new Error( + `Option ${arg} must have equal sign '=' separating it's argument`, + ); } } function throwForUnknownOption(arg: string): void { - for (const allowedOption of allowedPipOptions) { - if (arg.startsWith(allowedOption)) { + if (arg.includes('=')) { + const [option] = arg.split('='); + if (allowedPipOptions.includes(option)) { return; } } + if (allowedPipOptions.includes(arg)) { + return; + } throw new Error(`Option ${arg} not supported (yet)`); } From 8b2f4feb45435a656ae5cc5032743f6d8e4edbc3 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Thu, 8 Feb 2024 15:36:37 +0100 Subject: [PATCH 70/85] Safeguard for emit index url --- .../manager/pip-compile/artifacts.spec.ts | 38 ++++++++++++++----- lib/modules/manager/pip-compile/artifacts.ts | 10 +++-- lib/modules/manager/pip-compile/common.ts | 18 ++++++++- lib/modules/manager/pip-compile/types.ts | 2 + 4 files changed, 55 insertions(+), 13 deletions(-) diff --git a/lib/modules/manager/pip-compile/artifacts.spec.ts b/lib/modules/manager/pip-compile/artifacts.spec.ts index 685e65b89f85d8..5d4fda0c0c6488 100644 --- a/lib/modules/manager/pip-compile/artifacts.spec.ts +++ b/lib/modules/manager/pip-compile/artifacts.spec.ts @@ -22,12 +22,17 @@ jest.mock('../../../util/host-rules', () => mockDeep()); jest.mock('../../../util/http'); jest.mock('../../datasource', () => mockDeep()); -const simpleHeader = `# +function getCommandInHeader(command: string) { + return `# # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile requirements.in -#`; +# ${command} +# +`; +} + +const simpleHeader = getCommandInHeader('pip-compile requirements.in'); const adminConfig: RepoGlobalConfig = { // `join` fixes Windows CI @@ -89,7 +94,7 @@ describe('modules/manager/pip-compile/artifacts', () => { }), ).toBeNull(); expect(execSnapshots).toMatchObject([ - { cmd: 'pip-compile requirements.in' }, + { cmd: 'pip-compile --no-emit-index-url requirements.in' }, ]); }); @@ -134,7 +139,7 @@ describe('modules/manager/pip-compile/artifacts', () => { }), ).not.toBeNull(); expect(execSnapshots).toMatchObject([ - { cmd: 'pip-compile requirements.in' }, + { cmd: 'pip-compile --no-emit-index-url requirements.in' }, ]); }); @@ -182,7 +187,7 @@ describe('modules/manager/pip-compile/artifacts', () => { '&& ' + 'install-tool pip-tools 6.13.0 ' + '&& ' + - 'pip-compile requirements.in' + + 'pip-compile --no-emit-index-url requirements.in' + '"', }, ]); @@ -218,7 +223,7 @@ describe('modules/manager/pip-compile/artifacts', () => { { cmd: 'install-tool python 3.10.2' }, { cmd: 'install-tool pip-tools 6.13.0' }, { - cmd: 'pip-compile requirements.in', + cmd: 'pip-compile --no-emit-index-url requirements.in', options: { cwd: '/tmp/github/some/repo' }, }, ]); @@ -263,7 +268,7 @@ describe('modules/manager/pip-compile/artifacts', () => { }), ).not.toBeNull(); expect(execSnapshots).toMatchObject([ - { cmd: 'pip-compile requirements.in' }, + { cmd: 'pip-compile --no-emit-index-url requirements.in' }, ]); }); @@ -312,7 +317,7 @@ describe('modules/manager/pip-compile/artifacts', () => { '&& ' + 'install-tool pip-tools 6.13.0 ' + '&& ' + - 'pip-compile requirements.in' + + 'pip-compile --no-emit-index-url requirements.in' + '"', }, ]); @@ -339,6 +344,21 @@ describe('modules/manager/pip-compile/artifacts', () => { ); }); + it('safeguard against index url leak if not explicitly set by an option', () => { + expect( + constructPipCompileCmd(simpleHeader, 'subdir/requirements.txt'), + ).toBe('pip-compile --no-emit-index-url requirements.in'); + }); + + it('allow explicit --emit-index-url', () => { + expect( + constructPipCompileCmd( + getCommandInHeader('pip-compile --emit-index-url requirements.in'), + 'subdir/requirements.txt', + ), + ).toBe('pip-compile --emit-index-url requirements.in'); + }); + it('throws on unknown arguments', () => { expect(() => constructPipCompileCmd( diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index ed7a0350df1b17..6d7c3c4710c8a5 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -16,13 +16,17 @@ export function constructPipCompileCmd( outputFileName: string, strict: boolean = true, ): string { - const pipCompileArgs = extractHeaderCommand(content, outputFileName); - if (strict && pipCompileArgs.isCustomCommand) { + const headerArguments = extractHeaderCommand(content, outputFileName); + if (strict && headerArguments.isCustomCommand) { throw new Error( 'Detected custom command, header modified or set by CUSTOM_COMPILE_COMMAND', ); } - return pipCompileArgs.argv.map(quote).join(' '); + // safeguard against index url leak if not explicitly set by an option + if (!headerArguments.noEmitIndexUrl && !headerArguments.emitIndexUrl) { + headerArguments.argv.splice(1, 0, '--no-emit-index-url'); + } + return headerArguments.argv.map(quote).join(' '); } export async function updateArtifacts({ diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 300327e0f1e54f..36e6eb250d6eea 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -77,7 +77,8 @@ export const allowedPipOptions = [ '-v', '--allow-unsafe', '--generate-hashes', - '--no-emit-index-url', // TODO: handle this!!! + '--no-emit-index-url', + '--emit-index-url', '--strip-extras', '--index-url', ...optionsWithArguments, @@ -148,9 +149,21 @@ export function extractHeaderCommand( } else if (option === '--index-url') { result.indexUrl = value; // TODO: add to secrets? next PR + } else { + logger.warn(`pip-compile: option ${arg} not handled`); } continue; } + if (arg === '--no-emit-index-url') { + result.noEmitIndexUrl = true; + continue; + } + if (arg === '--emit-index-url') { + result.emitIndexUrl = true; + continue; + } + + logger.warn(`pip-compile: option ${arg} not handled`); } logger.trace( @@ -159,6 +172,9 @@ export function extractHeaderCommand( }, 'Parsed pip-compile command from header', ); + if (result.noEmitIndexUrl && result.emitIndexUrl) { + throw new Error('Cannot use both --no-emit-index-url and --emit-index-url'); + } if (result.sourceFiles.length === 0) { throw new Error( 'No source files detected in command, pass at least one package file explicitly', diff --git a/lib/modules/manager/pip-compile/types.ts b/lib/modules/manager/pip-compile/types.ts index 6feb336e2458c5..4d7ad9ad242ebd 100644 --- a/lib/modules/manager/pip-compile/types.ts +++ b/lib/modules/manager/pip-compile/types.ts @@ -13,6 +13,8 @@ export interface PipCompileArgs { extra?: string[]; extraIndexUrl?: string[]; indexUrl?: string; + noEmitIndexUrl?: boolean; + emitIndexUrl?: boolean; outputFile?: string; sourceFiles: string[]; // positional arguments } From 124ba2854cbcb6a4052ba17a1e0b1affef94bd14 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Thu, 8 Feb 2024 15:48:17 +0100 Subject: [PATCH 71/85] Extract getExecOptions method to common --- lib/modules/manager/pip-compile/artifacts.ts | 25 ++--------------- lib/modules/manager/pip-compile/common.ts | 28 ++++++++++++++++++++ 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index c62860ef329aa1..baad6550c798d4 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -3,10 +3,8 @@ import upath from 'upath'; import { TEMPORARY_ERROR } from '../../../constants/error-messages'; import { logger } from '../../../logger'; import { exec } from '../../../util/exec'; -import type { ExecOptions } from '../../../util/exec/types'; import { deleteLocalFile, - ensureCacheDir, readLocalFile, writeLocalFile, } from '../../../util/fs'; @@ -16,8 +14,7 @@ import type { UpdateArtifact, UpdateArtifactsResult } from '../types'; import { allowedPipArguments, constraintLineRegex, - getPipToolsConstraint, - getPythonConstraint, + getExecOptions, } from './common'; export function constructPipCompileCmd( @@ -86,25 +83,7 @@ export async function updateArtifacts({ inputFileName, outputFileName, ); - const constraint = getPythonConstraint(config); - const pipToolsConstraint = getPipToolsConstraint(config); - const execOptions: ExecOptions = { - cwdFile: inputFileName, - docker: {}, - toolConstraints: [ - { - toolName: 'python', - constraint, - }, - { - toolName: 'pip-tools', - constraint: pipToolsConstraint, - }, - ], - extraEnv: { - PIP_CACHE_DIR: await ensureCacheDir('pip'), - }, - }; + const execOptions = await getExecOptions(config, inputFileName); logger.trace({ cmd }, 'pip-compile command'); await exec(cmd, execOptions); const status = await getRepoStatus(); diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index d17dfdad7d390a..83d866fb0c97c9 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -1,5 +1,7 @@ import is from '@sindresorhus/is'; import { logger } from '../../../logger'; +import type { ExecOptions } from '../../../util/exec/types'; +import { ensureCacheDir } from '../../../util/fs'; import { regEx } from '../../../util/regex'; import type { UpdateArtifactsConfig } from '../types'; @@ -27,6 +29,32 @@ export function getPipToolsConstraint(config: UpdateArtifactsConfig): string { return ''; } +export async function getExecOptions( + config: UpdateArtifactsConfig, + inputFileName: string, +): Promise { + const constraint = getPythonConstraint(config); + const pipToolsConstraint = getPipToolsConstraint(config); + const execOptions: ExecOptions = { + cwdFile: inputFileName, + docker: {}, + toolConstraints: [ + { + toolName: 'python', + constraint, + }, + { + toolName: 'pip-tools', + constraint: pipToolsConstraint, + }, + ], + extraEnv: { + PIP_CACHE_DIR: await ensureCacheDir('pip'), + }, + }; + return execOptions; +} + export const constraintLineRegex = regEx( /^(#.*?\r?\n)+# {4}pip-compile(?.*?)\r?\n/, ); From 82773a88700916a22ffdcc35287a824c7785c896 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Thu, 8 Feb 2024 15:54:11 +0100 Subject: [PATCH 72/85] code move --- lib/modules/manager/pip-compile/common.ts | 50 +++++++++++------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 36e6eb250d6eea..8e3ae8aceef402 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -8,31 +8,6 @@ import { regEx } from '../../../util/regex'; import type { UpdateArtifactsConfig } from '../types'; import type { PipCompileArgs } from './types'; -export async function getExecOptions( - config: UpdateArtifactsConfig, - inputFileName: string, -): Promise { - const constraint = getPythonConstraint(config); - const pipToolsConstraint = getPipToolsConstraint(config); - const execOptions: ExecOptions = { - cwdFile: inputFileName, - docker: {}, - toolConstraints: [ - { - toolName: 'python', - constraint, - }, - { - toolName: 'pip-tools', - constraint: pipToolsConstraint, - }, - ], - extraEnv: { - PIP_CACHE_DIR: await ensureCacheDir('pip'), - }, - }; - return execOptions; -} export function getPythonConstraint( config: UpdateArtifactsConfig, ): string | undefined | null { @@ -58,6 +33,31 @@ export function getPipToolsConstraint(config: UpdateArtifactsConfig): string { return ''; } +export async function getExecOptions( + config: UpdateArtifactsConfig, + inputFileName: string, +): Promise { + const constraint = getPythonConstraint(config); + const pipToolsConstraint = getPipToolsConstraint(config); + const execOptions: ExecOptions = { + cwdFile: inputFileName, + docker: {}, + toolConstraints: [ + { + toolName: 'python', + constraint, + }, + { + toolName: 'pip-tools', + constraint: pipToolsConstraint, + }, + ], + extraEnv: { + PIP_CACHE_DIR: await ensureCacheDir('pip'), + }, + }; + return execOptions; +} export const constraintLineRegex = regEx( /^(#.*?\r?\n)+# {4}(?\S*)(? .*?)?\r?\n/, From 1516c612849fa7d7461a87a5c0c30879bc30fd82 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Fri, 9 Feb 2024 12:41:28 +0100 Subject: [PATCH 73/85] unused --- lib/modules/manager/pip-compile/artifacts.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index fc4bdb1ec5d2a6..807fb2f2fd4015 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -54,7 +54,6 @@ export function constructPipCompileCmd( export async function updateArtifacts({ packageFileName: inputFileName, newPackageFileContent: newInputContent, - updatedDeps, config, }: UpdateArtifact): Promise { if (!config.lockFiles) { From bee09da47c4b95a7f5e5ef088d1d95b36e6cc971 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Fri, 9 Feb 2024 12:42:11 +0100 Subject: [PATCH 74/85] unused strict --- lib/modules/manager/pip-compile/artifacts.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index 807fb2f2fd4015..4d2712b69c8009 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -15,10 +15,9 @@ import { extractHeaderCommand, getExecOptions } from './common'; export function constructPipCompileCmd( content: string, outputFileName: string, - strict: boolean = true, ): string { const headerArguments = extractHeaderCommand(content, outputFileName); - if (strict && headerArguments.isCustomCommand) { + if (headerArguments.isCustomCommand) { throw new Error( 'Detected custom command, header modified or set by CUSTOM_COMPILE_COMMAND', ); From 9945e7bd86701846edf7dca7478fedadc20681a7 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Fri, 9 Feb 2024 12:47:26 +0100 Subject: [PATCH 75/85] Add test --- lib/modules/manager/pip-compile/artifacts.spec.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/modules/manager/pip-compile/artifacts.spec.ts b/lib/modules/manager/pip-compile/artifacts.spec.ts index 5d4fda0c0c6488..8242b3975fe30e 100644 --- a/lib/modules/manager/pip-compile/artifacts.spec.ts +++ b/lib/modules/manager/pip-compile/artifacts.spec.ts @@ -359,6 +359,20 @@ describe('modules/manager/pip-compile/artifacts', () => { ).toBe('pip-compile --emit-index-url requirements.in'); }); + // TODO(not7cd): remove when relative pahts are supported + it('change --output-file if differs', () => { + expect( + constructPipCompileCmd( + getCommandInHeader( + 'pip-compile --output-file=hey.txt requirements.in', + ), + 'subdir/requirements.txt', + ), + ).toBe( + 'pip-compile --no-emit-index-url --output-file=requirements.txt requirements.in', + ); + }); + it('throws on unknown arguments', () => { expect(() => constructPipCompileCmd( From 7350c8ce0f54ca9891798ee822c2f263292d5e1f Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Fri, 9 Feb 2024 14:45:17 +0100 Subject: [PATCH 76/85] Fix test --- lib/modules/manager/pip-compile/artifacts.spec.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/modules/manager/pip-compile/artifacts.spec.ts b/lib/modules/manager/pip-compile/artifacts.spec.ts index 8242b3975fe30e..4c91e788cb786d 100644 --- a/lib/modules/manager/pip-compile/artifacts.spec.ts +++ b/lib/modules/manager/pip-compile/artifacts.spec.ts @@ -382,12 +382,11 @@ describe('modules/manager/pip-compile/artifacts', () => { ).toThrow(/supported/); }); - it('throws on custom command when strict', () => { + it('throws on custom command', () => { expect(() => constructPipCompileCmd( Fixtures.get('requirementsCustomCommand.txt'), 'subdir/requirements.txt', - true, ), ).toThrow(/custom/); }); From 3ddd94a12cae411e967bfbb876816d74e5a3e59a Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Mon, 12 Feb 2024 13:46:50 +0100 Subject: [PATCH 77/85] rename packageFileContent --- lib/modules/manager/pip-compile/extract.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index 9ec25404586cf2..11f9a0b3fc03f9 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -142,10 +142,14 @@ export async function extractAllPackageFiles( continue; } - const deps = extractPackageFile(content, packageFile, config); - if (deps) { + const packageFileContent = extractPackageFile( + content, + packageFile, + config, + ); + if (packageFileContent) { packageFiles.set(packageFile, { - ...deps, + ...packageFileContent, lockFiles: [fileMatch], packageFile, }); From 4b6d8f0c6d3e5b8e121a56cf9f8c71b2303e7261 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Mon, 12 Feb 2024 16:25:54 +0100 Subject: [PATCH 78/85] move --- lib/modules/manager/pip-compile/common.ts | 19 ++++++++++++++++++- lib/modules/manager/pip-compile/extract.ts | 20 +------------------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index e9383c594b3a86..193a46aa84eb19 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -5,7 +5,7 @@ import type { ExecOptions } from '../../../util/exec/types'; import { ensureCacheDir } from '../../../util/fs'; import { regEx } from '../../../util/regex'; import type { UpdateArtifactsConfig } from '../types'; -import type { PipCompileArgs } from './types'; +import type { DependencyBetweenFiles, PipCompileArgs } from './types'; export function getPythonConstraint( config: UpdateArtifactsConfig, @@ -210,3 +210,20 @@ function throwForUnknownOption(arg: string): void { } throw new Error(`Option ${arg} not supported (yet)`); } +export function generateMermaidGraph( + depsBetweenFiles: DependencyBetweenFiles[], + lockFileArgs: Map, +): string { + const lockFiles = []; + for (const lockFile of lockFileArgs.keys()) { + // TODO: add extra args to the lock file ${extraArgs ? '\n' + extraArgs : ''} + // const extraArgs = pipCompileArgs.extra + // ?.map((v) => '--extra=' + v) + // .join('\n'); + lockFiles.push(` ${lockFile}[[${lockFile}]]`); + } + const edges = depsBetweenFiles.map(({ sourceFile, outputFile, type }) => { + return ` ${sourceFile} -${type === 'constraint' ? '.' : ''}-> ${outputFile}`; + }); + return `graph TD\n${lockFiles.join('\n')}\n${edges.join('\n')}`; +} diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index 11f9a0b3fc03f9..882312499453ec 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -5,7 +5,7 @@ import { extractPackageFile as extractRequirementsFile } from '../pip_requiremen // import { extractPackageFile as extractSetupPyFile } from '../pip_setup'; // import { extractPackageFile as extractSetupCfgFile } from '../setup-cfg'; import type { ExtractConfig, PackageFile, PackageFileContent } from '../types'; -import { extractHeaderCommand } from './common'; +import { extractHeaderCommand, generateMermaidGraph } from './common'; import type { DependencyBetweenFiles, PipCompileArgs, @@ -60,24 +60,6 @@ export function extractPackageFile( } } -function generateMermaidGraph( - depsBetweenFiles: DependencyBetweenFiles[], - lockFileArgs: Map, -): string { - const lockFiles = []; - for (const lockFile of lockFileArgs.keys()) { - // TODO: add extra args to the lock file ${extraArgs ? '\n' + extraArgs : ''} - // const extraArgs = pipCompileArgs.extra - // ?.map((v) => '--extra=' + v) - // .join('\n'); - lockFiles.push(` ${lockFile}[[${lockFile}]]`); - } - const edges = depsBetweenFiles.map(({ sourceFile, outputFile, type }) => { - return ` ${sourceFile} -${type === 'constraint' ? '.' : ''}-> ${outputFile}`; - }); - return `graph TD\n${lockFiles.join('\n')}\n${edges.join('\n')}`; -} - export async function extractAllPackageFiles( config: ExtractConfig, fileMatches: string[], From 20a57b3a89e810e30868a67b692a5e15abb1d8b8 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Mon, 12 Feb 2024 16:26:04 +0100 Subject: [PATCH 79/85] remove unused test --- .../manager/pip-compile/artifacts.spec.ts | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/lib/modules/manager/pip-compile/artifacts.spec.ts b/lib/modules/manager/pip-compile/artifacts.spec.ts index 4c91e788cb786d..84b1798dffe6cd 100644 --- a/lib/modules/manager/pip-compile/artifacts.spec.ts +++ b/lib/modules/manager/pip-compile/artifacts.spec.ts @@ -390,21 +390,5 @@ describe('modules/manager/pip-compile/artifacts', () => { ), ).toThrow(/custom/); }); - - // TODO(not7cd): check for explotiable commands - // it('skips exploitable subcommands and files', () => { - // expect( - // constructPipCompileCmd( - // Fixtures.get('requirementsWithExploitingArguments.txt'), - // 'subdir/requirements.txt', - // ), - // ).toBe( - // 'pip-compile --generate-hashes --output-file=requirements.txt requirements.in', - // ); - // expect(logger.warn).toHaveBeenCalledWith( - // { argument: '--output-file=/etc/shadow' }, - // 'pip-compile was previously executed with an unexpected `--output-file` filename', - // ); - // }); }); }); From e3eaa751753e9ad23f27cd1bfbf0187029dbe160 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Mon, 12 Feb 2024 16:55:15 +0100 Subject: [PATCH 80/85] Add fileMatch migration --- lib/config/migration.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/config/migration.ts b/lib/config/migration.ts index 84822fc8633ff2..644133c25e9ef8 100644 --- a/lib/config/migration.ts +++ b/lib/config/migration.ts @@ -150,6 +150,16 @@ export function migrateConfig(config: RenovateConfig): MigratedConfig { } } } + if ( + is.nonEmptyObject(migratedConfig['pip-compile']) && + is.nonEmptyArray(migratedConfig['pip-compile'].fileMatch) + ) { + migratedConfig['pip-compile'].fileMatch = migratedConfig[ + 'pip-compile' + ].fileMatch.map((fileMatch) => { + return fileMatch.replace(/\.in\$$/, '.txt$'); + }); + } if (is.nonEmptyArray(migratedConfig.matchManagers)) { if (migratedConfig.matchManagers.includes('gradle-lite')) { if (!migratedConfig.matchManagers.includes('gradle')) { From 26169ebfdb980f0b07bb93d0ddbee3bc4437f5e7 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Mon, 12 Feb 2024 17:02:15 +0100 Subject: [PATCH 81/85] cast to string --- lib/config/migration.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/config/migration.ts b/lib/config/migration.ts index 644133c25e9ef8..b4b032c95acadc 100644 --- a/lib/config/migration.ts +++ b/lib/config/migration.ts @@ -157,7 +157,7 @@ export function migrateConfig(config: RenovateConfig): MigratedConfig { migratedConfig['pip-compile'].fileMatch = migratedConfig[ 'pip-compile' ].fileMatch.map((fileMatch) => { - return fileMatch.replace(/\.in\$$/, '.txt$'); + return (fileMatch as string).replace(/\.in\$$/, '.txt$'); }); } if (is.nonEmptyArray(migratedConfig.matchManagers)) { From b03ce91ce9d063dfd2b1523f29e8e692353f774a Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Mon, 12 Feb 2024 17:07:51 +0100 Subject: [PATCH 82/85] match all cases --- lib/config/migration.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/config/migration.ts b/lib/config/migration.ts index b4b032c95acadc..67f7d7a9035233 100644 --- a/lib/config/migration.ts +++ b/lib/config/migration.ts @@ -157,7 +157,11 @@ export function migrateConfig(config: RenovateConfig): MigratedConfig { migratedConfig['pip-compile'].fileMatch = migratedConfig[ 'pip-compile' ].fileMatch.map((fileMatch) => { - return (fileMatch as string).replace(/\.in\$$/, '.txt$'); + const match = fileMatch as string; + if (match.endsWith('.in')) { + return match.replace(/\.in$/, '.txt'); + } + return match.replace(/\.in\$$/, '.txt$'); }); } if (is.nonEmptyArray(migratedConfig.matchManagers)) { From 52d2378fbcbf70f4773633d21ab42eaf25cd2d96 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Mon, 12 Feb 2024 17:20:33 +0100 Subject: [PATCH 83/85] Test migration --- .../__snapshots__/migration.spec.ts.snap | 16 +++++++++++++++ lib/config/migration.spec.ts | 20 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/lib/config/__snapshots__/migration.spec.ts.snap b/lib/config/__snapshots__/migration.spec.ts.snap index b3f0b3deb66552..ee173d49ae74e9 100644 --- a/lib/config/__snapshots__/migration.spec.ts.snap +++ b/lib/config/__snapshots__/migration.spec.ts.snap @@ -79,6 +79,22 @@ exports[`config/migration it migrates nested packageRules 1`] = ` } `; +exports[`config/migration it migrates pip-compile 1`] = ` +{ + "pip-compile": { + "enabled": true, + "fileMatch": [ + "(^|/)requirements\\.txt$", + "(^|/)requirements-fmt\\.txt$", + "(^|/)requirements-lint\\.txt$", + ".github/workflows/requirements.txt", + "(^|/)debian_packages/private/third_party/requirements\\.txt$", + "(^|/).*?requirements.*?\\.txt$", + ], + }, +} +`; + exports[`config/migration migrateConfig(config, parentConfig) does not migrate multi days 1`] = ` { "schedule": "after 5:00pm on wednesday and thursday", diff --git a/lib/config/migration.spec.ts b/lib/config/migration.spec.ts index 86889d5d6011ff..ae9977e8e36171 100644 --- a/lib/config/migration.spec.ts +++ b/lib/config/migration.spec.ts @@ -668,6 +668,26 @@ describe('config/migration', () => { expect(migratedConfig).toMatchSnapshot(); }); + it('it migrates pip-compile', () => { + const config: RenovateConfig = { + 'pip-compile': { + enabled: true, + fileMatch: [ + '(^|/)requirements\\.in$', + '(^|/)requirements-fmt\\.in$', + '(^|/)requirements-lint\\.in$', + '.github/workflows/requirements.in', + '(^|/)debian_packages/private/third_party/requirements\\.in$', + '(^|/).*?requirements.*?\\.in$', + ], + }, + }; + const { isMigrated, migratedConfig } = + configMigration.migrateConfig(config); + expect(isMigrated).toBeTrue(); + expect(migratedConfig).toMatchSnapshot(); + }); + it('it migrates gradle-lite', () => { const config: RenovateConfig = { 'gradle-lite': { From 4c3f2ffc5f5e5a31ddbac2455a8f9c77432fbe11 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Mon, 12 Feb 2024 19:06:49 +0100 Subject: [PATCH 84/85] Remove test for no header --- lib/modules/manager/pip-compile/artifacts.spec.ts | 10 ---------- lib/modules/manager/pip-compile/common.ts | 10 +++++++--- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/lib/modules/manager/pip-compile/artifacts.spec.ts b/lib/modules/manager/pip-compile/artifacts.spec.ts index e9918850595701..a22f9aca75dd55 100644 --- a/lib/modules/manager/pip-compile/artifacts.spec.ts +++ b/lib/modules/manager/pip-compile/artifacts.spec.ts @@ -334,16 +334,6 @@ describe('modules/manager/pip-compile/artifacts', () => { ).toThrow(/extract/); }); - it('returns --no-emit-index-url when credentials are present in URLs', () => { - expect( - constructPipCompileCmd( - Fixtures.get('requirementsNoHeaders.txt'), - 'subdir/requirements.txt', - true, - ), - ).toBe('pip-compile --no-emit-index-url requirements.in'); - }); - it('returns extracted common arguments (like those featured in the README)', () => { expect( constructPipCompileCmd( diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index f994d5c793c0a8..cef57dee2b9c3a 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -7,7 +7,11 @@ import { ensureCacheDir } from '../../../util/fs'; import * as hostRules from '../../../util/host-rules'; import { regEx } from '../../../util/regex'; import type { PackageFileContent, UpdateArtifactsConfig } from '../types'; -import type { GetRegistryUrlVarsResult, PipCompileArgs, DependencyBetweenFiles } from './types'; +import type { + DependencyBetweenFiles, + GetRegistryUrlVarsResult, + PipCompileArgs, +} from './types'; export function getPythonConstraint( config: UpdateArtifactsConfig, @@ -193,7 +197,6 @@ function throwForDisallowedOption(arg: string): void { throw new Error(`Option ${arg} not allowed for this manager`); } } - function throwForNoEqualSignInOptionWithArgument(arg: string): void { if (optionsWithArguments.includes(arg)) { throw new Error( @@ -201,7 +204,6 @@ function throwForNoEqualSignInOptionWithArgument(arg: string): void { ); } } - function throwForUnknownOption(arg: string): void { if (arg.includes('=')) { const [option] = arg.split('='); @@ -214,6 +216,7 @@ function throwForUnknownOption(arg: string): void { } throw new Error(`Option ${arg} not supported (yet)`); } + export function generateMermaidGraph( depsBetweenFiles: DependencyBetweenFiles[], lockFileArgs: Map, @@ -230,6 +233,7 @@ export function generateMermaidGraph( return ` ${sourceFile} -${type === 'constraint' ? '.' : ''}-> ${outputFile}`; }); return `graph TD\n${lockFiles.join('\n')}\n${edges.join('\n')}`; +} function buildRegistryUrl(url: string): URL | null { try { From 04fe1231a7e6c86d0b3a3ed618bdca1c4fb738b7 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Tue, 13 Feb 2024 11:50:44 +0100 Subject: [PATCH 85/85] remove use of toMatchSnapshot --- lib/config/__snapshots__/migration.spec.ts.snap | 16 ---------------- lib/config/migration.spec.ts | 14 +++++++++++++- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/lib/config/__snapshots__/migration.spec.ts.snap b/lib/config/__snapshots__/migration.spec.ts.snap index ee173d49ae74e9..b3f0b3deb66552 100644 --- a/lib/config/__snapshots__/migration.spec.ts.snap +++ b/lib/config/__snapshots__/migration.spec.ts.snap @@ -79,22 +79,6 @@ exports[`config/migration it migrates nested packageRules 1`] = ` } `; -exports[`config/migration it migrates pip-compile 1`] = ` -{ - "pip-compile": { - "enabled": true, - "fileMatch": [ - "(^|/)requirements\\.txt$", - "(^|/)requirements-fmt\\.txt$", - "(^|/)requirements-lint\\.txt$", - ".github/workflows/requirements.txt", - "(^|/)debian_packages/private/third_party/requirements\\.txt$", - "(^|/).*?requirements.*?\\.txt$", - ], - }, -} -`; - exports[`config/migration migrateConfig(config, parentConfig) does not migrate multi days 1`] = ` { "schedule": "after 5:00pm on wednesday and thursday", diff --git a/lib/config/migration.spec.ts b/lib/config/migration.spec.ts index ae9977e8e36171..baa90a4aa50dbc 100644 --- a/lib/config/migration.spec.ts +++ b/lib/config/migration.spec.ts @@ -685,7 +685,19 @@ describe('config/migration', () => { const { isMigrated, migratedConfig } = configMigration.migrateConfig(config); expect(isMigrated).toBeTrue(); - expect(migratedConfig).toMatchSnapshot(); + expect(migratedConfig).toEqual({ + 'pip-compile': { + enabled: true, + fileMatch: [ + '(^|/)requirements\\.txt$', + '(^|/)requirements-fmt\\.txt$', + '(^|/)requirements-lint\\.txt$', + '.github/workflows/requirements.txt', + '(^|/)debian_packages/private/third_party/requirements\\.txt$', + '(^|/).*?requirements.*?\\.txt$', + ], + }, + }); }); it('it migrates gradle-lite', () => {