From 2f4c7111b033898c89180d5526efe2b50c5d4bdb Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Thu, 31 Aug 2023 10:53:22 +0300 Subject: [PATCH] feat(docker): Support `releaseTimestamp` for DockerHub tag results (#24164) Co-authored-by: Michael Kriese --- lib/modules/datasource/docker/index.spec.ts | 25 +++++- lib/modules/datasource/docker/index.ts | 98 +++++++++++++-------- lib/modules/datasource/docker/schema.ts | 35 ++++++++ 3 files changed, 118 insertions(+), 40 deletions(-) diff --git a/lib/modules/datasource/docker/index.spec.ts b/lib/modules/datasource/docker/index.spec.ts index 6e86d9c1f78165..ff1c4289c66853 100644 --- a/lib/modules/datasource/docker/index.spec.ts +++ b/lib/modules/datasource/docker/index.spec.ts @@ -1561,11 +1561,21 @@ describe('modules/datasource/docker/index', () => { .get('/library/node/tags?page_size=1000') .reply(200, { next: `${dockerHubUrl}/library/node/tags?page=2&page_size=1000`, - results: [{ name: '1.0.0' }], + results: [ + { + name: '1.0.0', + tag_last_pushed: '2021-01-01T00:00:00.000Z', + }, + ], }) .get('/library/node/tags?page=2&page_size=1000') .reply(200, { - results: [{ name: '0.9.0' }], + results: [ + { + name: '0.9.0', + tag_last_pushed: '2020-01-01T00:00:00.000Z', + }, + ], }); httpMock .scope(baseUrl) @@ -1577,7 +1587,16 @@ describe('modules/datasource/docker/index', () => { datasource: DockerDatasource.id, packageName: 'docker.io/node', }); - expect(res?.releases).toHaveLength(2); + expect(res?.releases).toMatchObject([ + { + version: '0.9.0', + releaseTimestamp: '2020-01-01T00:00:00.000Z', + }, + { + version: '1.0.0', + releaseTimestamp: '2021-01-01T00:00:00.000Z', + }, + ]); }); it('adds no library/ prefix for other registries', async () => { diff --git a/lib/modules/datasource/docker/index.ts b/lib/modules/datasource/docker/index.ts index 6a975cac06ca74..42f4d44e89f5b2 100644 --- a/lib/modules/datasource/docker/index.ts +++ b/lib/modules/datasource/docker/index.ts @@ -7,6 +7,7 @@ import { HttpError } from '../../../util/http'; import type { HttpResponse } from '../../../util/http/types'; import { hasKey } from '../../../util/object'; import { regEx } from '../../../util/regex'; +import { type AsyncResult, Result } from '../../../util/result'; import { isDockerDigest } from '../../../util/string'; import { ensurePathPrefix, @@ -15,7 +16,12 @@ import { } from '../../../util/url'; import { id as dockerVersioningId } from '../../versioning/docker'; import { Datasource } from '../datasource'; -import type { DigestConfig, GetReleasesConfig, ReleaseResult } from '../types'; +import type { + DigestConfig, + GetReleasesConfig, + Release, + ReleaseResult, +} from '../types'; import { isArtifactoryServer } from '../util'; import { DOCKER_HUB, @@ -29,8 +35,11 @@ import { sourceLabels, } from './common'; import { ecrPublicRegex, ecrRegex, isECRMaxResultsError } from './ecr'; -import type { Manifest, OciImageConfig } from './schema'; -import type { DockerHubTags } from './types'; +import { + DockerHubTagsPage, + type Manifest, + type OciImageConfig, +} from './schema'; const defaultConfig = { commitMessageTopic: '{{{depName}}} Docker tag', @@ -820,34 +829,26 @@ export class DockerDatasource extends Datasource { return digest; } - async getDockerHubTags(dockerRepository: string): Promise { - if (!process.env.RENOVATE_X_DOCKER_HUB_TAGS) { - return null; - } - try { - let index = 0; - let tags: string[] = []; - let url: - | string - | undefined = `https://hub.docker.com/v2/repositories/${dockerRepository}/tags?page_size=1000`; - do { - const res: DockerHubTags = (await this.http.getJson(url)) - .body; - tags = tags.concat(res.results.map((tag) => tag.name)); - url = res.next; - index += 1; - } while (url && index < 100); - logger.debug( - `getDockerHubTags(${dockerRepository}): found ${tags.length} tags` - ); - return tags; - } catch (err) { - logger.debug( - { dockerRepository, errMessage: err.message }, - `No Docker Hub tags result - falling back to docker.io api` - ); + async getDockerHubTags(dockerRepository: string): Promise { + const result: Release[] = []; + let url: + | null + | string = `https://hub.docker.com/v2/repositories/${dockerRepository}/tags?page_size=1000`; + while (url) { + const { val, err } = await this.http + .getJsonSafe(url, DockerHubTagsPage) + .unwrap(); + + if (err) { + logger.debug({ err }, `Docker: error fetching data from DockerHub`); + return null; + } + + result.push(...val.items); + url = val.nextPage; } - return null; + + return result; } /** @@ -883,20 +884,43 @@ export class DockerDatasource extends Datasource { packageName, registryUrl! ); - let tags: string[] | null = null; - if (registryHost === 'https://index.docker.io') { - tags = await this.getDockerHubTags(dockerRepository); - } - tags ??= await this.getTags(registryHost, dockerRepository); - if (!tags) { + + type TagsResultType = AsyncResult< + Release[], + NonNullable + >; + + const getTags = (): TagsResultType => + Result.wrapNullable( + this.getTags(registryHost, dockerRepository), + 'tags-error' as const + ).transform((tags) => tags.map((version) => ({ version }))); + + const getDockerHubTags = (): TagsResultType => + Result.wrapNullable( + this.getDockerHubTags(dockerRepository), + 'dockerhub-error' as const + ).catch(getTags); + + const tagsResult = + registryHost === 'https://index.docker.io' && + process.env.RENOVATE_X_DOCKER_HUB_TAGS + ? getDockerHubTags() + : getTags(); + + const { val: releases, err } = await tagsResult.unwrap(); + if (err instanceof Error) { + throw err; + } else if (err) { return null; } - const releases = tags.map((version) => ({ version })); + const ret: ReleaseResult = { registryUrl: registryHost, releases, }; + const tags = releases.map((release) => release.version); const latestTag = tags.includes('latest') ? 'latest' : findLatestStable(tags) ?? tags[tags.length - 1]; diff --git a/lib/modules/datasource/docker/schema.ts b/lib/modules/datasource/docker/schema.ts index 43e51b0cfe51b6..6fe542c79fd03f 100644 --- a/lib/modules/datasource/docker/schema.ts +++ b/lib/modules/datasource/docker/schema.ts @@ -1,4 +1,7 @@ import { z } from 'zod'; +import { logger } from '../../../logger'; +import { LooseArray } from '../../../util/schema-utils'; +import type { Release } from '../types'; // Helm manifests export const HelmConfigBlob = z.object({ @@ -127,3 +130,35 @@ export const Manifest = z.union([ ]); export type Manifest = z.infer; + +export const DockerHubTag = z + .object({ + name: z.string(), + tag_last_pushed: z.string().datetime().nullable().catch(null), + }) + .transform(({ name, tag_last_pushed }) => { + const release: Release = { version: name }; + + if (tag_last_pushed) { + release.releaseTimestamp = tag_last_pushed; + } + + return release; + }); + +export const DockerHubTagsPage = z + .object({ + next: z.string().nullable().catch(null), + results: LooseArray(DockerHubTag, { + onError: /* istanbul ignore next */ ({ error }) => { + logger.debug( + { error }, + 'Docker: Failed to parse some tags from Docker Hub' + ); + }, + }), + }) + .transform(({ next, results }) => ({ + nextPage: next, + items: results, + }));