diff --git a/lib/modules/manager/pip-compile/extract.spec.ts b/lib/modules/manager/pip-compile/extract.spec.ts index bd45db12fff388..533348a7571c8f 100644 --- a/lib/modules/manager/pip-compile/extract.spec.ts +++ b/lib/modules/manager/pip-compile/extract.spec.ts @@ -246,26 +246,32 @@ describe('modules/manager/pip-compile/extract', () => { }); it('return sorted package files', async () => { - fs.readLocalFile.mockResolvedValueOnce( - getSimpleRequirementsFile('pip-compile --output-file=4.txt 3.in', [ - 'foo==1.0.1', - ]), - ); - fs.readLocalFile.mockResolvedValueOnce('-r 2.txt\nfoo'); - fs.readLocalFile.mockResolvedValueOnce( - getSimpleRequirementsFile('pip-compile --output-file=2.txt 1.in', [ - 'foo==1.0.1', - ]), - ); - fs.readLocalFile.mockResolvedValueOnce('foo'); + fs.readLocalFile.mockImplementation((name): any => { + if (name === '1.in') { + return 'foo'; + } else if (name === '2.txt') { + return getSimpleRequirementsFile( + 'pip-compile --output-file=2.txt 1.in', + ['foo==1.0.1'], + ); + } else if (name === '3.in') { + return '-r 2.txt\nfoo'; + } else if (name === '4.txt') { + return getSimpleRequirementsFile( + 'pip-compile --output-file=4.txt 3.in', + ['foo==1.0.1'], + ); + } + return null; + }); const lockFiles = ['4.txt', '2.txt']; const packageFiles = await extractAllPackageFiles({}, lockFiles); expect(packageFiles).toBeDefined(); expect(packageFiles?.map((p) => p.packageFile)).toEqual(['1.in', '3.in']); - expect(packageFiles?.map((p) => p.lockFiles!.pop())).toEqual([ - '2.txt', - '4.txt', + expect(packageFiles?.map((p) => p.lockFiles)).toEqual([ + ['2.txt', '4.txt'], + ['4.txt'], ]); }); @@ -359,4 +365,112 @@ describe('modules/manager/pip-compile/extract', () => { 'pip-compile: dependency not found in lock file', ); }); + + it('handles -r reference to another input file', async () => { + fs.readLocalFile.mockImplementation((name): any => { + if (name === '1.in') { + return 'foo'; + } else if (name === '2.txt') { + return getSimpleRequirementsFile( + 'pip-compile --output-file=2.txt 1.in', + ['foo==1.0.1'], + ); + } else if (name === '3.in') { + return '-r 1.in\nfoo'; + } else if (name === '4.txt') { + return getSimpleRequirementsFile( + 'pip-compile --output-file=4.txt 3.in', + ['foo==1.0.1'], + ); + } + return null; + }); + + const lockFiles = ['4.txt', '2.txt']; + const packageFiles = await extractAllPackageFiles({}, lockFiles); + expect(packageFiles?.map((p) => p.lockFiles)).toEqual([ + ['2.txt', '4.txt'], + ['4.txt'], + ]); + }); + + it('handles transitive -r references', async () => { + fs.readLocalFile.mockImplementation((name): any => { + if (name === '1.in') { + return 'foo'; + } else if (name === '2.txt') { + return getSimpleRequirementsFile( + 'pip-compile --output-file=2.txt 1.in', + ['foo==1.0.1'], + ); + } else if (name === '3.in') { + return '-r 1.in\nfoo'; + } else if (name === '4.txt') { + return getSimpleRequirementsFile( + 'pip-compile --output-file=4.txt 3.in', + ['foo==1.0.1'], + ); + } else if (name === '5.in') { + return '-r 4.txt\nfoo'; + } else if (name === '6.txt') { + return getSimpleRequirementsFile( + 'pip-compile --output-file=6.txt 5.in', + ['foo==1.0.1'], + ); + } + return null; + }); + + const lockFiles = ['4.txt', '2.txt', '6.txt']; + const packageFiles = await extractAllPackageFiles({}, lockFiles); + expect(packageFiles?.map((p) => p.lockFiles)).toEqual([ + ['2.txt', '4.txt', '6.txt'], + ['4.txt', '6.txt'], + ['6.txt'], + ]); + }); + + it('warns on -r reference to failed file', async () => { + fs.readLocalFile.mockImplementation((name): any => { + if (name === 'reqs-no-headers.txt') { + return Fixtures.get('requirementsNoHeaders.txt'); + } else if (name === '1.in') { + return '-r reqs-no-headers.txt\nfoo'; + } else if (name === '2.txt') { + return getSimpleRequirementsFile( + 'pip-compile --output-file=2.txt 1.in', + ['foo==1.0.1'], + ); + } + return null; + }); + + const lockFiles = ['reqs-no-headers.txt', '2.txt']; + const packageFiles = await extractAllPackageFiles({}, lockFiles); + expect(packageFiles?.map((p) => p.lockFiles)).toEqual([['2.txt']]); + expect(logger.warn).toHaveBeenCalledWith( + 'pip-compile: 1.in references reqs-no-headers.txt which does not appear to be a requirements file managed by pip-compile', + ); + }); + + it('warns on -r reference to requirements file not managed by pip-compile', async () => { + fs.readLocalFile.mockImplementation((name): any => { + if (name === '1.in') { + return '-r unmanaged-file.txt\nfoo'; + } else if (name === '2.txt') { + return getSimpleRequirementsFile( + 'pip-compile --output-file=2.txt 1.in', + ['foo==1.0.1'], + ); + } + return null; + }); + + const lockFiles = ['2.txt']; + const packageFiles = await extractAllPackageFiles({}, lockFiles); + expect(packageFiles?.map((p) => p.lockFiles)).toEqual([['2.txt']]); + expect(logger.warn).toHaveBeenCalledWith( + 'pip-compile: 1.in references unmanaged-file.txt which does not appear to be a requirements file managed by pip-compile', + ); + }); }); diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index ed011867f7ad02..b3b8730c24f788 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -70,6 +70,7 @@ export async function extractAllPackageFiles( const lockFileArgs = new Map(); const depsBetweenFiles: DependencyBetweenFiles[] = []; const packageFiles = new Map(); + const lockFileSources = new Map(); for (const fileMatch of fileMatches) { const fileContent = await readLocalFile(fileMatch, 'utf8'); if (!fileContent) { @@ -132,7 +133,9 @@ export async function extractAllPackageFiles( logger.debug( `pip-compile: ${packageFile} used in multiple output files`, ); - packageFiles.get(packageFile)!.lockFiles!.push(fileMatch); + const existingPackageFile = packageFiles.get(packageFile)!; + existingPackageFile.lockFiles!.push(fileMatch); + lockFileSources.set(fileMatch, existingPackageFile); continue; } const content = await readLocalFile(packageFile, 'utf8'); @@ -180,11 +183,13 @@ export async function extractAllPackageFiles( ); } } - packageFiles.set(packageFile, { + const newPackageFile: PackageFile = { ...packageFileContent, lockFiles: [fileMatch], packageFile, - }); + }; + packageFiles.set(packageFile, newPackageFile); + lockFileSources.set(fileMatch, newPackageFile); } else { logger.warn( { packageFile }, @@ -200,6 +205,25 @@ export async function extractAllPackageFiles( depsBetweenFiles, packageFiles, ); + + // This needs to go in reverse order to handle transitive dependencies + for (const packageFile of [...result].reverse()) { + for (const reqFile of packageFile.managerData?.requirementsFiles ?? []) { + let sourceFile: PackageFile | undefined = undefined; + if (fileMatches.includes(reqFile)) { + sourceFile = lockFileSources.get(reqFile); + } else if (packageFiles.has(reqFile)) { + sourceFile = packageFiles.get(reqFile); + } + if (!sourceFile) { + logger.warn( + `pip-compile: ${packageFile.packageFile} references ${reqFile} which does not appear to be a requirements file managed by pip-compile`, + ); + continue; + } + sourceFile.lockFiles!.push(...packageFile.lockFiles!); + } + } logger.debug( 'pip-compile: dependency graph:\n' + generateMermaidGraph(depsBetweenFiles, lockFileArgs),