Skip to content

Commit

Permalink
feat(manager/pip-compile): Support -r dependencies between files (#28052
Browse files Browse the repository at this point in the history 
)

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
  • Loading branch information
@mbudnek @viceice
mbudnek and viceice committed Mar 29, 2024
1 parent e998c65 commit f9ba846
Show file tree
Hide file tree
Showing 2 changed files with 156 additions and 18 deletions.
144 changes: 129 additions & 15 deletions lib/modules/manager/pip-compile/extract.spec.ts
View file
Expand Up @@ -246,26 +246,32 @@ describe('modules/manager/pip-compile/extract', () => {
});

it('return sorted package files', async () => {
fs.readLocalFile.mockResolvedValueOnce(
getSimpleRequirementsFile('pip-compile --output-file=4.txt 3.in', [
'foo==1.0.1',
]),
);
fs.readLocalFile.mockResolvedValueOnce('-r 2.txt\nfoo');
fs.readLocalFile.mockResolvedValueOnce(
getSimpleRequirementsFile('pip-compile --output-file=2.txt 1.in', [
'foo==1.0.1',
]),
);
fs.readLocalFile.mockResolvedValueOnce('foo');
fs.readLocalFile.mockImplementation((name): any => {
if (name === '1.in') {
return 'foo';
} else if (name === '2.txt') {
return getSimpleRequirementsFile(
'pip-compile --output-file=2.txt 1.in',
['foo==1.0.1'],
);
} else if (name === '3.in') {
return '-r 2.txt\nfoo';
} else if (name === '4.txt') {
return getSimpleRequirementsFile(
'pip-compile --output-file=4.txt 3.in',
['foo==1.0.1'],
);
}
return null;
});

const lockFiles = ['4.txt', '2.txt'];
const packageFiles = await extractAllPackageFiles({}, lockFiles);
expect(packageFiles).toBeDefined();
expect(packageFiles?.map((p) => p.packageFile)).toEqual(['1.in', '3.in']);
expect(packageFiles?.map((p) => p.lockFiles!.pop())).toEqual([
'2.txt',
'4.txt',
expect(packageFiles?.map((p) => p.lockFiles)).toEqual([
['2.txt', '4.txt'],
['4.txt'],
]);
});

Expand Down Expand Up @@ -359,4 +365,112 @@ describe('modules/manager/pip-compile/extract', () => {
'pip-compile: dependency not found in lock file',
);
});

it('handles -r reference to another input file', async () => {
fs.readLocalFile.mockImplementation((name): any => {
if (name === '1.in') {
return 'foo';
} else if (name === '2.txt') {
return getSimpleRequirementsFile(
'pip-compile --output-file=2.txt 1.in',
['foo==1.0.1'],
);
} else if (name === '3.in') {
return '-r 1.in\nfoo';
} else if (name === '4.txt') {
return getSimpleRequirementsFile(
'pip-compile --output-file=4.txt 3.in',
['foo==1.0.1'],
);
}
return null;
});

const lockFiles = ['4.txt', '2.txt'];
const packageFiles = await extractAllPackageFiles({}, lockFiles);
expect(packageFiles?.map((p) => p.lockFiles)).toEqual([
['2.txt', '4.txt'],
['4.txt'],
]);
});

it('handles transitive -r references', async () => {
fs.readLocalFile.mockImplementation((name): any => {
if (name === '1.in') {
return 'foo';
} else if (name === '2.txt') {
return getSimpleRequirementsFile(
'pip-compile --output-file=2.txt 1.in',
['foo==1.0.1'],
);
} else if (name === '3.in') {
return '-r 1.in\nfoo';
} else if (name === '4.txt') {
return getSimpleRequirementsFile(
'pip-compile --output-file=4.txt 3.in',
['foo==1.0.1'],
);
} else if (name === '5.in') {
return '-r 4.txt\nfoo';
} else if (name === '6.txt') {
return getSimpleRequirementsFile(
'pip-compile --output-file=6.txt 5.in',
['foo==1.0.1'],
);
}
return null;
});

const lockFiles = ['4.txt', '2.txt', '6.txt'];
const packageFiles = await extractAllPackageFiles({}, lockFiles);
expect(packageFiles?.map((p) => p.lockFiles)).toEqual([
['2.txt', '4.txt', '6.txt'],
['4.txt', '6.txt'],
['6.txt'],
]);
});

it('warns on -r reference to failed file', async () => {
fs.readLocalFile.mockImplementation((name): any => {
if (name === 'reqs-no-headers.txt') {
return Fixtures.get('requirementsNoHeaders.txt');
} else if (name === '1.in') {
return '-r reqs-no-headers.txt\nfoo';
} else if (name === '2.txt') {
return getSimpleRequirementsFile(
'pip-compile --output-file=2.txt 1.in',
['foo==1.0.1'],
);
}
return null;
});

const lockFiles = ['reqs-no-headers.txt', '2.txt'];
const packageFiles = await extractAllPackageFiles({}, lockFiles);
expect(packageFiles?.map((p) => p.lockFiles)).toEqual([['2.txt']]);
expect(logger.warn).toHaveBeenCalledWith(
'pip-compile: 1.in references reqs-no-headers.txt which does not appear to be a requirements file managed by pip-compile',
);
});

it('warns on -r reference to requirements file not managed by pip-compile', async () => {
fs.readLocalFile.mockImplementation((name): any => {
if (name === '1.in') {
return '-r unmanaged-file.txt\nfoo';
} else if (name === '2.txt') {
return getSimpleRequirementsFile(
'pip-compile --output-file=2.txt 1.in',
['foo==1.0.1'],
);
}
return null;
});

const lockFiles = ['2.txt'];
const packageFiles = await extractAllPackageFiles({}, lockFiles);
expect(packageFiles?.map((p) => p.lockFiles)).toEqual([['2.txt']]);
expect(logger.warn).toHaveBeenCalledWith(
'pip-compile: 1.in references unmanaged-file.txt which does not appear to be a requirements file managed by pip-compile',
);
});
});
30 changes: 27 additions & 3 deletions lib/modules/manager/pip-compile/extract.ts
View file
Expand Up @@ -70,6 +70,7 @@ export async function extractAllPackageFiles(
const lockFileArgs = new Map<string, PipCompileArgs>();
const depsBetweenFiles: DependencyBetweenFiles[] = [];
const packageFiles = new Map<string, PackageFile>();
const lockFileSources = new Map<string, PackageFile>();
for (const fileMatch of fileMatches) {
const fileContent = await readLocalFile(fileMatch, 'utf8');
if (!fileContent) {
Expand Down Expand Up @@ -132,7 +133,9 @@ export async function extractAllPackageFiles(
logger.debug(
`pip-compile: ${packageFile} used in multiple output files`,
);
packageFiles.get(packageFile)!.lockFiles!.push(fileMatch);
const existingPackageFile = packageFiles.get(packageFile)!;
existingPackageFile.lockFiles!.push(fileMatch);
lockFileSources.set(fileMatch, existingPackageFile);
continue;
}
const content = await readLocalFile(packageFile, 'utf8');
Expand Down Expand Up @@ -180,11 +183,13 @@ export async function extractAllPackageFiles(
);
}
}
packageFiles.set(packageFile, {
const newPackageFile: PackageFile = {
...packageFileContent,
lockFiles: [fileMatch],
packageFile,
});
};
packageFiles.set(packageFile, newPackageFile);
lockFileSources.set(fileMatch, newPackageFile);
} else {
logger.warn(
{ packageFile },
Expand All @@ -200,6 +205,25 @@ export async function extractAllPackageFiles(
depsBetweenFiles,
packageFiles,
);

// This needs to go in reverse order to handle transitive dependencies
for (const packageFile of [...result].reverse()) {
for (const reqFile of packageFile.managerData?.requirementsFiles ?? []) {
let sourceFile: PackageFile | undefined = undefined;
if (fileMatches.includes(reqFile)) {
sourceFile = lockFileSources.get(reqFile);
} else if (packageFiles.has(reqFile)) {
sourceFile = packageFiles.get(reqFile);
}
if (!sourceFile) {
logger.warn(
`pip-compile: ${packageFile.packageFile} references ${reqFile} which does not appear to be a requirements file managed by pip-compile`,
);
continue;
}
sourceFile.lockFiles!.push(...packageFile.lockFiles!);
}
}
logger.debug(
'pip-compile: dependency graph:\n' +
generateMermaidGraph(depsBetweenFiles, lockFileArgs),
Expand Down

0 comments on commit f9ba846

Please sign in to comment.