feat(manager/pip-compile): Add lockedVersion to package file deps (#2…

…7242)

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
Co-authored-by: Rhys Arkins <rhys@arkins.net>

lib/modules/datasource/pypi/common.ts

@@ -5,3 +5,8 @@ const githubRepoPattern = regEx(/^https?:\/\/github\.com\/[^/]+\/[^/]+$/);
 export function isGitHubRepo(url: string): boolean {
   return !url.includes('sponsors') && githubRepoPattern.test(url);
 }
+
+// https://packaging.python.org/en/latest/specifications/name-normalization/
+export function normalizeDepName(name: string): string {
+  return name.replace(/[-_.]+/g, '-').toLowerCase();
+}

lib/modules/datasource/pypi/index.ts

@@ -8,7 +8,7 @@ import { ensureTrailingSlash } from '../../../util/url';
 import * as pep440 from '../../versioning/pep440';
 import { Datasource } from '../datasource';
 import type { GetReleasesConfig, Release, ReleaseResult } from '../types';
-import { isGitHubRepo } from './common';
+import { isGitHubRepo, normalizeDepName } from './common';
 import type { PypiJSON, PypiJSONRelease, Releases } from './types';
 
 export class PypiDatasource extends Datasource {
@@ -79,17 +79,13 @@ export class PypiDatasource extends Datasource {
     return input.toLowerCase().replace(regEx(/_/g), '-');
   }
 
-  private static normalizeNameForUrlLookup(input: string): string {
-    return input.toLowerCase().replace(regEx(/(_|\.|-)+/g), '-');
-  }
-
   private async getDependency(
     packageName: string,
     hostUrl: string,
   ): Promise<ReleaseResult | null> {
     const lookupUrl = url.resolve(
       hostUrl,
-      `${PypiDatasource.normalizeNameForUrlLookup(packageName)}/json`,
+      `${normalizeDepName(packageName)}/json`,
     );
     const dependency: ReleaseResult = { releases: [] };
     logger.trace({ lookupUrl }, 'Pypi api got lookup');
@@ -227,9 +223,7 @@ export class PypiDatasource extends Datasource {
   ): Promise<ReleaseResult | null> {
     const lookupUrl = url.resolve(
       hostUrl,
-      ensureTrailingSlash(
-        PypiDatasource.normalizeNameForUrlLookup(packageName),
-      ),
+      ensureTrailingSlash(normalizeDepName(packageName)),
     );
     const dependency: ReleaseResult = { releases: [] };
     const response = await this.http.get(lookupUrl);

lib/modules/manager/pip-compile/extract.spec.ts

@@ -1,5 +1,6 @@
 import { Fixtures } from '../../../../test/fixtures';
 import { fs } from '../../../../test/util';
+import { logger } from '../../../logger';
 import { extractAllPackageFiles, extractPackageFile } from '.';
 
 jest.mock('../../../util/fs');
@@ -139,11 +140,64 @@ describe('modules/manager/pip-compile/extract', () => {
         ['foo==1.0.1'],
       ),
     );
-    fs.readLocalFile.mockResolvedValueOnce('!@#$');
-    fs.readLocalFile.mockResolvedValueOnce('');
+    fs.readLocalFile.mockResolvedValueOnce('!@#$'); // malformed.in
+    fs.readLocalFile.mockResolvedValueOnce(''); // empty.in
+    fs.readLocalFile.mockResolvedValueOnce(
+      getSimpleRequirementsFile(
+        'pip-compile --output-file=headerOnly.txt reqs.in',
+        [],
+      ),
+    );
 
-    const lockFiles = ['empty.txt', 'noHeader.txt', 'badSource.txt'];
+    const lockFiles = [
+      'empty.txt',
+      'noHeader.txt',
+      'badSource.txt',
+      'headerOnly.txt',
+    ];
     const packageFiles = await extractAllPackageFiles({}, lockFiles);
     expect(packageFiles).toBeNull();
   });
+
+  it('adds lockedVersion to deps in package file', async () => {
+    fs.readLocalFile.mockResolvedValueOnce(
+      getSimpleRequirementsFile(
+        'pip-compile --output-file=requirements.txt requirements.in',
+        ['friendly-bard==1.0.1'],
+      ),
+    );
+    // also check if normalized name is used
+    fs.readLocalFile.mockResolvedValueOnce('FrIeNdLy-._.-bArD>=1.0.0');
+
+    const lockFiles = ['requirements.txt'];
+    const packageFiles = await extractAllPackageFiles({}, lockFiles);
+    expect(packageFiles).toBeDefined();
+    const packageFile = packageFiles!.pop();
+    expect(packageFile!.deps).toHaveLength(1);
+    expect(packageFile!.deps.pop()).toMatchObject({
+      currentValue: '>=1.0.0',
+      depName: 'FrIeNdLy-._.-bArD',
+      lockedVersion: '1.0.1',
+    });
+  });
+
+  it('warns if dependency has no locked version', async () => {
+    fs.readLocalFile.mockResolvedValueOnce(
+      getSimpleRequirementsFile(
+        'pip-compile --output-file=requirements.txt requirements.in',
+        ['foo==1.0.1'],
+      ),
+    );
+    fs.readLocalFile.mockResolvedValueOnce('foo>=1.0.0\nbar');
+
+    const lockFiles = ['requirements.txt'];
+    const packageFiles = await extractAllPackageFiles({}, lockFiles);
+    expect(packageFiles).toBeDefined();
+    const packageFile = packageFiles!.pop();
+    expect(packageFile!.deps).toHaveLength(2);
+    expect(logger.warn).toHaveBeenCalledWith(
+      { depName: 'bar', lockFile: 'requirements.txt' },
+      'pip-compile: dependency not found in lock file',
+    );
+  });
 });

lib/modules/manager/pip-compile/extract.ts

@@ -1,5 +1,6 @@
 import { logger } from '../../../logger';
 import { readLocalFile } from '../../../util/fs';
+import { normalizeDepName } from '../../datasource/pypi/common';
 import { extractPackageFile as extractRequirementsFile } from '../pip_requirements/extract';
 import { extractPackageFile as extractSetupPyFile } from '../pip_setup';
 import type { ExtractConfig, PackageFile, PackageFileContent } from '../types';
@@ -87,8 +88,15 @@ export async function extractAllPackageFiles(
         type: 'constraint',
       });
     }
-    // TODO(not7cd): handle locked deps
-    // const lockedDeps = extractRequirementsFile(content);
+    const lockedDeps = extractRequirementsFile(fileContent)?.deps;
+    if (!lockedDeps) {
+      logger.debug(
+        { fileMatch },
+        'pip-compile: Failed to extract dependencies from lock file',
+      );
+      continue;
+    }
+
     for (const packageFile of compileArgs.sourceFiles) {
       depsBetweenFiles.push({
         sourceFile: packageFile,
@@ -122,6 +130,21 @@ export async function extractAllPackageFiles(
         config,
       );
       if (packageFileContent) {
+        for (const dep of packageFileContent.deps) {
+          const lockedVersion = lockedDeps?.find(
+            (lockedDep) =>
+              normalizeDepName(lockedDep.depName!) ===
+              normalizeDepName(dep.depName!),
+          )?.currentVersion;
+          if (lockedVersion) {
+            dep.lockedVersion = lockedVersion;
+          } else {
+            logger.warn(
+              { depName: dep.depName, lockFile: fileMatch },
+              'pip-compile: dependency not found in lock file',
+            );
+          }
+        }
         packageFiles.set(packageFile, {
           ...packageFileContent,
           lockFiles: [fileMatch],