feat(datasource): Add python-version datasource (#27583)

Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com>
Co-authored-by: Rhys Arkins <rhys@arkins.net>
Co-authored-by: Michael Kriese <michael.kriese@visualon.de>

lib/modules/datasource/api.ts

@@ -51,6 +51,7 @@ import { PackagistDatasource } from './packagist';
 import { PodDatasource } from './pod';
 import { PuppetForgeDatasource } from './puppet-forge';
 import { PypiDatasource } from './pypi';
+import { PythonVersionDatasource } from './python-version';
 import { RepologyDatasource } from './repology';
 import { RubyVersionDatasource } from './ruby-version';
 import { RubyGemsDatasource } from './rubygems';
@@ -120,6 +121,7 @@ api.set(PackagistDatasource.id, new PackagistDatasource());
 api.set(PodDatasource.id, new PodDatasource());
 api.set(PuppetForgeDatasource.id, new PuppetForgeDatasource());
 api.set(PypiDatasource.id, new PypiDatasource());
+api.set(PythonVersionDatasource.id, new PythonVersionDatasource());
 api.set(RepologyDatasource.id, new RepologyDatasource());
 api.set(RubyVersionDatasource.id, new RubyVersionDatasource());
 api.set(RubyGemsDatasource.id, new RubyGemsDatasource());

lib/modules/datasource/python-version/__fixtures__/eol.json

@@ -0,0 +1,4 @@
+[
+  {"cycle":"3.12","releaseDate":"2023-10-02","support":"2025-04-02","eol":"2028-10-31","latest":"3.12.2","latestReleaseDate":"2024-02-06","lts":false},
+  {"cycle":"3.7","releaseDate":"2018-06-26","support":"2020-06-27","eol":"2023-06-27","latest":"3.7.17","latestReleaseDate":"2023-06-05","lts":false}
+]

lib/modules/datasource/python-version/__fixtures__/release.json

@@ -0,0 +1,7 @@
+[
+  {"name": "Python 3.12.0", "slug": "python-3120", "version": 3, "is_published": true, "is_latest": false, "release_date": "2023-10-02T12:50:09Z", "pre_release": false, "release_page": null, "release_notes_url": "https://docs.python.org/release/3.12.0/whatsnew/changelog.html#python-3-12-0", "show_on_download_page": true, "resource_uri": "https://www.python.org/api/v2/downloads/release/832/"},
+  {"name": "Python 3.12.0a1", "slug": "python-3120a1", "version": 3, "is_published": true, "is_latest": false, "release_date": "2022-10-25T02:16:12Z", "pre_release": true, "release_page": null, "release_notes_url": "", "show_on_download_page": false, "resource_uri": "https://www.python.org/api/v2/downloads/release/767/"},
+  {"name": "Python 3.12.2", "slug": "python-3122", "version": 3, "is_published": true, "is_latest": true, "release_date": "2024-02-06T21:40:35Z", "pre_release": false, "release_page": null, "release_notes_url": "https://docs.python.org/release/3.12.2/whatsnew/changelog.html#python-3-12-2", "show_on_download_page": true, "resource_uri": "https://www.python.org/api/v2/downloads/release/871/"},
+  {"name": "Python 3.7.8", "slug": "python-378", "version": 3, "is_published": true, "is_latest": false, "release_date": "2020-06-27T12:55:01Z", "pre_release": false, "release_page": null, "release_notes_url": "https://docs.python.org/release/3.7.8/whatsnew/changelog.html#changelog", "show_on_download_page": true, "resource_uri": "https://www.python.org/api/v2/downloads/release/442/"},
+  {"name": "Python 3.7.9", "slug": "python-379", "version": 3, "is_published": true, "is_latest": false, "release_date": "2020-08-17T22:00:00Z", "pre_release": false, "release_page": null, "release_notes_url": "https://docs.python.org/release/3.7.9/whatsnew/changelog.html#changelog", "show_on_download_page": true, "resource_uri": "https://www.python.org/api/v2/downloads/release/482/"}
+]

lib/modules/datasource/python-version/common.ts

@@ -0,0 +1,5 @@
+export const defaultRegistryUrl =
+  'https://www.python.org/api/v2/downloads/release';
+export const githubBaseUrl = 'https://api.github.com/';
+
+export const datasource = 'python-version';

lib/modules/datasource/python-version/index.spec.ts

@@ -0,0 +1,148 @@
+import { satisfies } from '@renovatebot/pep440';
+import { getPkgReleases } from '..';
+import { Fixtures } from '../../../../test/fixtures';
+import * as httpMock from '../../../../test/http-mock';
+import { EXTERNAL_HOST_ERROR } from '../../../constants/error-messages';
+import * as githubGraphql from '../../../util/github/graphql';
+import { registryUrl as eolRegistryUrl } from '../endoflife-date/common';
+import { datasource, defaultRegistryUrl } from './common';
+import { PythonVersionDatasource } from '.';
+
+describe('modules/datasource/python-version/index', () => {
+  describe('dependent datasources', () => {
+    it('returns Python EOL data', async () => {
+      const datasource = new PythonVersionDatasource();
+      httpMock
+        .scope(eolRegistryUrl)
+        .get('/python.json')
+        .reply(200, Fixtures.get('eol.json'));
+      const res = await datasource.getEolReleases();
+      expect(
+        res?.releases.find((release) => release.version === '3.7.17')
+          ?.isDeprecated,
+      ).toBeTrue();
+    });
+  });
+
+  describe('getReleases', () => {
+    beforeEach(() => {
+      httpMock
+        .scope('https://endoflife.date')
+        .get('/api/python.json')
+        .reply(200, Fixtures.get('eol.json'));
+
+      jest.spyOn(githubGraphql, 'queryReleases').mockResolvedValueOnce([
+        {
+          id: 1,
+          url: 'https://example.com',
+          name: 'containerbase/python-prebuild',
+          description: 'some description',
+          version: '3.12.1',
+          releaseTimestamp: '2020-03-09T13:00:00Z',
+        },
+        {
+          id: 2,
+          url: 'https://example.com',
+          name: 'containerbase/python-prebuild',
+          description: 'some description',
+          version: '3.12.0',
+          releaseTimestamp: '2020-03-09T13:00:00Z',
+        },
+        {
+          id: 3,
+          url: 'https://example.com',
+          name: 'containerbase/python-prebuild',
+          description: 'some description',
+          version: '3.7.8',
+          releaseTimestamp: '2020-03-09T13:00:00Z',
+        },
+      ]);
+    });
+
+    it('throws for 500', async () => {
+      httpMock.scope(defaultRegistryUrl).get('').reply(500);
+      await expect(
+        getPkgReleases({
+          datasource,
+          packageName: 'python',
+        }),
+      ).rejects.toThrow(EXTERNAL_HOST_ERROR);
+    });
+
+    it('returns null for error', async () => {
+      httpMock.scope(defaultRegistryUrl).get('').replyWithError('error');
+      expect(
+        await getPkgReleases({
+          datasource,
+          packageName: 'python',
+        }),
+      ).toBeNull();
+    });
+
+    it('returns null for empty 200 OK', async () => {
+      httpMock.scope(defaultRegistryUrl).get('').reply(200, []);
+      expect(
+        await getPkgReleases({
+          datasource,
+          packageName: 'python',
+        }),
+      ).toBeNull();
+    });
+
+    describe('processes real data', () => {
+      beforeEach(() => {
+        httpMock
+          .scope(defaultRegistryUrl)
+          .get('')
+          .reply(200, Fixtures.get('release.json'));
+      });
+
+      it('returns the correct data', async () => {
+        const res = await getPkgReleases({
+          datasource,
+          packageName: 'python',
+        });
+        expect(res?.releases[0]).toEqual({
+          isDeprecated: true,
+          isStable: true,
+          releaseTimestamp: '2020-06-27T12:55:01.000Z',
+          version: '3.7.8',
+        });
+      });
+
+      it('only returns stable versions', async () => {
+        const res = await getPkgReleases({
+          datasource,
+          packageName: 'python',
+        });
+        expect(res?.releases).toHaveLength(2);
+        for (const release of res?.releases ?? []) {
+          expect(release.isStable).toBeTrue();
+        }
+      });
+
+      it('only returns versions that are prebuilt', async () => {
+        const res = await getPkgReleases({
+          datasource,
+          packageName: 'python',
+        });
+        expect(
+          res?.releases.filter((release) =>
+            satisfies(release.version, '>3.12.1'),
+          ),
+        ).toHaveLength(0);
+      });
+
+      it('returns isDeprecated status for Python 3 minor releases', async () => {
+        const res = await getPkgReleases({
+          datasource,
+          packageName: 'python',
+        });
+        expect(res?.releases).toHaveLength(2);
+        for (const release of res?.releases ?? []) {
+          expect(release.isDeprecated).toBeBoolean();
+        }
+      });
+    });
+  });
+});

lib/modules/datasource/python-version/index.ts

@@ -0,0 +1,92 @@
+import { cache } from '../../../util/cache/package/decorator';
+import { id as versioning } from '../../versioning/python';
+import { Datasource } from '../datasource';
+import { EndoflifeDatePackagesource } from '../endoflife-date';
+import { registryUrl as eolRegistryUrl } from '../endoflife-date/common';
+import { GithubReleasesDatasource } from '../github-releases';
+import type { GetReleasesConfig, ReleaseResult } from '../types';
+import { datasource, defaultRegistryUrl, githubBaseUrl } from './common';
+import { PythonRelease } from './schema';
+
+export class PythonVersionDatasource extends Datasource {
+  static readonly id = datasource;
+  pythonPrebuildDatasource: GithubReleasesDatasource;
+  pythonEolDatasource: EndoflifeDatePackagesource;
+
+  constructor() {
+    super(datasource);
+    this.pythonPrebuildDatasource = new GithubReleasesDatasource();
+    this.pythonEolDatasource = new EndoflifeDatePackagesource();
+  }
+
+  override readonly customRegistrySupport = false;
+
+  override readonly defaultRegistryUrls = [defaultRegistryUrl];
+
+  override readonly defaultVersioning = versioning;
+
+  override readonly caching = true;
+
+  async getPrebuildReleases(): Promise<ReleaseResult | null> {
+    return await this.pythonPrebuildDatasource.getReleases({
+      registryUrl: githubBaseUrl,
+      packageName: 'containerbase/python-prebuild',
+    });
+  }
+
+  async getEolReleases(): Promise<ReleaseResult | null> {
+    return await this.pythonEolDatasource.getReleases({
+      registryUrl: eolRegistryUrl,
+      packageName: 'python',
+    });
+  }
+
+  @cache({
+    namespace: `datasource-${datasource}`,
+    key: ({ registryUrl }: GetReleasesConfig) => `${registryUrl}`,
+  })
+  async getReleases({
+    registryUrl,
+  }: GetReleasesConfig): Promise<ReleaseResult | null> {
+    // istanbul ignore if
+    if (!registryUrl) {
+      return null;
+    }
+    const pythonPrebuildReleases = await this.getPrebuildReleases();
+    const pythonPrebuildVersions = new Set<string>(
+      pythonPrebuildReleases?.releases.map((release) => release.version),
+    );
+    const pythonEolReleases = await this.getEolReleases();
+    const pythonEolVersions = new Map(
+      pythonEolReleases?.releases
+        .filter((release) => release.isDeprecated !== undefined)
+        .map((release) => [
+          release.version.split('.').slice(0, 2).join('.'),
+          release.isDeprecated,
+        ]),
+    );
+    const result: ReleaseResult = {
+      homepage: 'https://python.org',
+      sourceUrl: 'https://github.com/python/cpython',
+      registryUrl,
+      releases: [],
+    };
+    try {
+      const response = await this.http.getJson(registryUrl, PythonRelease);
+      result.releases.push(
+        ...response.body
+          .filter((release) => release.isStable)
+          .filter((release) => pythonPrebuildVersions.has(release.version)),
+      );
+    } catch (err) {
+      this.handleGenericErrors(err);
+    }
+    for (const release of result.releases) {
+      release.isDeprecated = pythonEolVersions.get(
+        release.version.split('.').slice(0, 2).join('.'),
+      );
+    }
+
+    return result.releases.length ? result : null;
+  }
+}

lib/modules/datasource/python-version/readme.md

@@ -0,0 +1,36 @@
+This datasource returns Python releases from the [python.org API](https://www.python.org/api/v2/downloads/release/).
+
+It also fetches deprecated versions from the [Endoflife Date datasource](../endoflife-date/index.md).
+
+Because Renovate depends on [`containerbase/python-prebuild`](https://github.com/containerbase/python-prebuild/releases) it will also fetch releases from the GitHub API.
+
+## Example custom manager
+
+Below is a [custom regex manager](../../manager/regex/index.md) to update the Python versions in a Dockerfile.
+Python versions sometimes drop the dot that separate the major and minor number: so `3.11` becomes `311`.
+The example below handles this case.
+
+```dockerfile
+ARG PYTHON_VERSION=311
+FROM image-python${PYTHON_VERSION}-builder:1.0.0
+```
+
+```json
+{
+  "customManagers": [
+    {
+      "customType": "regex",
+      "fileMatch": ["^Dockerfile$"],
+      "matchStringsStrategy": "any",
+      "matchStrings": [
+        "ARG PYTHON_VERSION=\"?(?<currentValue>3(?<minor>\\d+))\"?\\s"
+      ],
+      "autoReplaceStringTemplate": "ARG PYTHON_VERSION={{{replace '\\.' '' newValue}}}\n",
+      "currentValueTemplate": "3.{{{minor}}}",
+      "datasourceTemplate": "python-version",
+      "versioningTemplate": "python",
+      "depNameTemplate": "python"
+    }
+  ]
+}
+```

lib/modules/datasource/python-version/schema.ts

@@ -0,0 +1,31 @@
+import { z } from 'zod';
+import type { Release } from '../types';
+
+export const PythonRelease = z
+  .object({
+    /** e.g: "Python 3.9.0b1" */
+    name: z.string(),
+    /** e.g: "python-390b1" */
+    slug: z.string(),
+    /** Major version e.g: 3 */
+    version: z.number(),
+    /** is latest major version, true for Python 2.7.18 and latest Python 3 */
+    is_latest: z.boolean(),
+    is_published: z.boolean(),
+    release_date: z.string(),
+    pre_release: z.boolean(),
+    release_page: z.string().nullable(),
+    show_on_download_page: z.boolean(),
+    /** Changelog e.g: "https://docs.python.org/…html#python-3-9-0-beta-1" */
+    release_notes_url: z.string(),
+    /** Download URL e.g: "https://www.python.org/api/v2/downloads/release/436/" */
+    resource_uri: z.string(),
+  })
+  .transform(
+    ({ name, release_date: releaseTimestamp, pre_release }): Release => {
+      const version = name?.replace('Python', '').trim();
+      const isStable = pre_release === false;
+      return { version, releaseTimestamp, isStable };
+    },
+  )
+  .array();

lib/util/cache/package/types.ts

@@ -90,6 +90,7 @@ export type PackageCacheNamespace =
   | 'datasource-packagist-public-files'
   | 'datasource-packagist'
   | 'datasource-pod'
+  | 'datasource-python-version'
   | 'datasource-releases'
   | 'datasource-repology-list'
   | 'datasource-ruby-version'