From 30b3e0c2871d5076ceb8239463414f85e531bbcc Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Mon, 26 Feb 2024 15:17:42 +0100 Subject: [PATCH 1/6] Add indirect deps to PackageFile --- lib/modules/manager/pip-compile/extract.ts | 30 +++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index ed011867f7ad02..b8c7cdb6a49b2d 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -5,7 +5,12 @@ import { ensureLocalPath } from '../../../util/fs/util'; import { normalizeDepName } from '../../datasource/pypi/common'; import { extractPackageFile as extractRequirementsFile } from '../pip_requirements/extract'; import { extractPackageFile as extractSetupPyFile } from '../pip_setup'; -import type { ExtractConfig, PackageFile, PackageFileContent } from '../types'; +import type { + ExtractConfig, + PackageDependency, + PackageFile, + PackageFileContent, +} from '../types'; import { extractHeaderCommand } from './common'; import type { DependencyBetweenFiles, @@ -180,6 +185,17 @@ export async function extractAllPackageFiles( ); } } + for (const lockedDep of lockedDeps) { + if ( + !packageFileContent.deps.find( + (dep) => + normalizeDepName(lockedDep.depName!) === + normalizeDepName(dep.depName!), + ) + ) { + packageFileContent.deps.push(indirectDep(lockedDep)); + } + } packageFiles.set(packageFile, { ...packageFileContent, lockFiles: [fileMatch], @@ -206,3 +222,15 @@ export async function extractAllPackageFiles( ); return result; } + +function indirectDep(dep: PackageDependency): PackageDependency { + const result = { + ...dep, + lockedVersion: dep.currentVersion, + depType: 'indirect', + enabled: false, + }; + delete result.currentValue; + delete result.currentVersion; + return result; +} From 44cf0a8668f7d6ad20d30e9243e04b6cc35871e7 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Mon, 26 Feb 2024 15:52:20 +0100 Subject: [PATCH 2/6] Add indirect deps to already processed packageFiles --- lib/modules/manager/pip-compile/extract.ts | 32 ++++++++++++++-------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index b8c7cdb6a49b2d..935eacecdf2190 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -137,7 +137,9 @@ export async function extractAllPackageFiles( logger.debug( `pip-compile: ${packageFile} used in multiple output files`, ); - packageFiles.get(packageFile)!.lockFiles!.push(fileMatch); + const packageFileContent = packageFiles.get(packageFile)!; + packageFileContent.lockFiles!.push(fileMatch); + extendWithIndirectDeps(packageFileContent, lockedDeps); continue; } const content = await readLocalFile(packageFile, 'utf8'); @@ -185,17 +187,7 @@ export async function extractAllPackageFiles( ); } } - for (const lockedDep of lockedDeps) { - if ( - !packageFileContent.deps.find( - (dep) => - normalizeDepName(lockedDep.depName!) === - normalizeDepName(dep.depName!), - ) - ) { - packageFileContent.deps.push(indirectDep(lockedDep)); - } - } + extendWithIndirectDeps(packageFileContent, lockedDeps); packageFiles.set(packageFile, { ...packageFileContent, lockFiles: [fileMatch], @@ -223,6 +215,22 @@ export async function extractAllPackageFiles( return result; } +function extendWithIndirectDeps( + packageFileContent: PackageFileContent, + lockedDeps: PackageDependency[], +) { + for (const lockedDep of lockedDeps) { + if ( + !packageFileContent.deps.find( + (dep) => + normalizeDepName(lockedDep.depName!) === + normalizeDepName(dep.depName!), + ) + ) { + packageFileContent.deps.push(indirectDep(lockedDep)); + } + } +} function indirectDep(dep: PackageDependency): PackageDependency { const result = { ...dep, From 09d58757b283b743c95e34ebf3e5dbc2a75e7492 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Mon, 26 Feb 2024 16:18:33 +0100 Subject: [PATCH 3/6] Add a unit test --- .../manager/pip-compile/extract.spec.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/lib/modules/manager/pip-compile/extract.spec.ts b/lib/modules/manager/pip-compile/extract.spec.ts index bd45db12fff388..cd95b1d8c39343 100644 --- a/lib/modules/manager/pip-compile/extract.spec.ts +++ b/lib/modules/manager/pip-compile/extract.spec.ts @@ -359,4 +359,27 @@ describe('modules/manager/pip-compile/extract', () => { 'pip-compile: dependency not found in lock file', ); }); + + it('adds transitive dependency to deps in package file', async () => { + fs.readLocalFile.mockResolvedValueOnce( + getSimpleRequirementsFile( + 'pip-compile --output-file=requirements.txt requirements.in', + ['friendly-bard==1.0.1', 'bards-friend==1.0.0'], + ), + ); + fs.readLocalFile.mockResolvedValueOnce('FrIeNdLy-._.-bArD>=1.0.0'); + + const lockFiles = ['requirements.txt']; + const packageFiles = await extractAllPackageFiles({}, lockFiles); + expect(packageFiles).toBeDefined(); + const packageFile = packageFiles!.pop(); + expect(packageFile!.deps).toHaveLength(2); + expect(packageFile!.deps[1]).toEqual({ + datasource: 'pypi', + depType: 'indirect', + depName: 'bards-friend', + lockedVersion: '1.0.0', + enabled: false, + }); + }); }); From 8b1275b72c7a985ee77875ff9c29307a9e687b1f Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Mon, 26 Feb 2024 16:23:12 +0100 Subject: [PATCH 4/6] Return type --- lib/modules/manager/pip-compile/extract.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index 935eacecdf2190..c6283c96185c8c 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -218,7 +218,7 @@ export async function extractAllPackageFiles( function extendWithIndirectDeps( packageFileContent: PackageFileContent, lockedDeps: PackageDependency[], -) { +): void { for (const lockedDep of lockedDeps) { if ( !packageFileContent.deps.find( From dc63fac937ec91785b27cc073e0c491e367f7917 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Mon, 26 Feb 2024 16:30:22 +0100 Subject: [PATCH 5/6] Update docs --- lib/modules/manager/pip-compile/readme.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/modules/manager/pip-compile/readme.md b/lib/modules/manager/pip-compile/readme.md index 21719995a8d148..7c179203534b68 100644 --- a/lib/modules/manager/pip-compile/readme.md +++ b/lib/modules/manager/pip-compile/readme.md @@ -74,3 +74,9 @@ Renovate reads the `requirements.txt` file and extracts these `pip-compile` argu - `--output-file` All other allowed `pip-compile` arguments will be passed over without modification. + +### Transitive / indirect dependencies + +This manager detects dependencies that only appear in lock files. +They are disabled by default but can be forced to enable by vulnerability alerts. +They will be upgraded with `--upgrade-package` option. From 3aeebe2cfda11c855c697b0c17269193457db975 Mon Sep 17 00:00:00 2001 From: Norbert Szulc Date: Fri, 29 Mar 2024 22:51:08 +0100 Subject: [PATCH 6/6] Add JSDoc --- lib/modules/manager/pip-compile/extract.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index c6283c96185c8c..ec8ac2679e494c 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -231,6 +231,22 @@ function extendWithIndirectDeps( } } } + +/** + * As indirect dependecies don't exist in the package file, we need to + * create them from the lock file. + * + * By removing currentValue and currentVersion, we ensure that they + * are handled like unconstrained dependencies with locked version. + * Such packages are updated when their update strategy + * is set to 'update-lockfile', + * see: lib/workers/repository/process/lookup/index.ts. + * + * By disabling them by default, we won't create noise by updating them. + * Unless they have vulnerability alert, then they are forced to be updated. + * @param dep dependency extracted from lock file (requirements.txt) + * @returns unconstrained dependency with locked version + */ function indirectDep(dep: PackageDependency): PackageDependency { const result = { ...dep,