From d676ce5b1297373db3767235cb798d4ac4fd154a Mon Sep 17 00:00:00 2001 From: Samuel Dolt Date: Thu, 16 Sep 2021 12:47:30 +0200 Subject: [PATCH] feat(datasource): add Gitlab Package support (#11672) Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com> Co-authored-by: Michael Kriese Co-authored-by: Rhys Arkins --- lib/constants/platform.spec.ts | 4 + lib/constants/platforms.ts | 1 + lib/datasource/api.ts | 2 + .../__snapshots__/index.spec.ts.snap | 21 ++++ lib/datasource/gitlab-packages/common.ts | 1 + lib/datasource/gitlab-packages/index.spec.ts | 102 ++++++++++++++++++ lib/datasource/gitlab-packages/index.ts | 84 +++++++++++++++ lib/datasource/gitlab-packages/readme.md | 39 +++++++ lib/datasource/gitlab-packages/types.ts | 5 + 9 files changed, 259 insertions(+) create mode 100644 lib/datasource/gitlab-packages/__snapshots__/index.spec.ts.snap create mode 100644 lib/datasource/gitlab-packages/common.ts create mode 100644 lib/datasource/gitlab-packages/index.spec.ts create mode 100644 lib/datasource/gitlab-packages/index.ts create mode 100644 lib/datasource/gitlab-packages/readme.md create mode 100644 lib/datasource/gitlab-packages/types.ts diff --git a/lib/constants/platform.spec.ts b/lib/constants/platform.spec.ts index 22984c3e06143f..7b2f536b8d3e40 100644 --- a/lib/constants/platform.spec.ts +++ b/lib/constants/platform.spec.ts @@ -1,5 +1,6 @@ import { id as GH_RELEASES_DS } from '../datasource/github-releases'; import { id as GH_TAGS_DS } from '../datasource/github-tags'; +import { GitlabPackagesDatasource } from '../datasource/gitlab-packages'; import { GitlabReleasesDatasource } from '../datasource/gitlab-releases'; import { id as GL_TAGS_DS } from '../datasource/gitlab-tags'; import { id as POD_DS } from '../datasource/pod'; @@ -16,6 +17,9 @@ describe('constants/platform', () => { expect( GITLAB_API_USING_HOST_TYPES.includes(GitlabReleasesDatasource.id) ).toBeTrue(); + expect( + GITLAB_API_USING_HOST_TYPES.includes(GitlabPackagesDatasource.id) + ).toBeTrue(); expect( GITLAB_API_USING_HOST_TYPES.includes(PLATFORM_TYPE_GITLAB) ).toBeTrue(); diff --git a/lib/constants/platforms.ts b/lib/constants/platforms.ts index 94725ae52da84d..0bee6d8191f78b 100644 --- a/lib/constants/platforms.ts +++ b/lib/constants/platforms.ts @@ -16,4 +16,5 @@ export const GITLAB_API_USING_HOST_TYPES = [ PLATFORM_TYPE_GITLAB, 'gitlab-releases', 'gitlab-tags', + 'gitlab-packages', ]; diff --git a/lib/datasource/api.ts b/lib/datasource/api.ts index 342bb6f2202e94..33631ce7be1537 100644 --- a/lib/datasource/api.ts +++ b/lib/datasource/api.ts @@ -11,6 +11,7 @@ import * as gitRefs from './git-refs'; import * as gitTags from './git-tags'; import * as githubReleases from './github-releases'; import * as githubTags from './github-tags'; +import { GitlabPackagesDatasource } from './gitlab-packages'; import { GitlabReleasesDatasource } from './gitlab-releases'; import * as gitlabTags from './gitlab-tags'; import * as go from './go'; @@ -51,6 +52,7 @@ api.set('git-refs', gitRefs); api.set('git-tags', gitTags); api.set('github-releases', githubReleases); api.set('github-tags', githubTags); +api.set('gitlab-packages', new GitlabPackagesDatasource()); api.set('gitlab-tags', gitlabTags); api.set(GitlabReleasesDatasource.id, new GitlabReleasesDatasource()); api.set('go', go); diff --git a/lib/datasource/gitlab-packages/__snapshots__/index.spec.ts.snap b/lib/datasource/gitlab-packages/__snapshots__/index.spec.ts.snap new file mode 100644 index 00000000000000..b1a7880be88661 --- /dev/null +++ b/lib/datasource/gitlab-packages/__snapshots__/index.spec.ts.snap @@ -0,0 +1,21 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`datasource/gitlab-packages/index getReleases returns package from custom registry 1`] = ` +Object { + "registryUrl": "https://gitlab.com", + "releases": Array [ + Object { + "releaseTimestamp": "2020-03-04T18:01:37.000Z", + "version": "1.0.0", + }, + Object { + "releaseTimestamp": "2020-04-04T18:01:37.000Z", + "version": "v1.1.0", + }, + Object { + "releaseTimestamp": "2020-05-04T18:01:37.000Z", + "version": "v1.1.1", + }, + ], +} +`; diff --git a/lib/datasource/gitlab-packages/common.ts b/lib/datasource/gitlab-packages/common.ts new file mode 100644 index 00000000000000..22ebcad0662afc --- /dev/null +++ b/lib/datasource/gitlab-packages/common.ts @@ -0,0 +1 @@ +export const datasource = 'gitlab-packages'; diff --git a/lib/datasource/gitlab-packages/index.spec.ts b/lib/datasource/gitlab-packages/index.spec.ts new file mode 100644 index 00000000000000..7710f4ea8478ab --- /dev/null +++ b/lib/datasource/gitlab-packages/index.spec.ts @@ -0,0 +1,102 @@ +import { getPkgReleases } from '..'; +import * as httpMock from '../../../test/http-mock'; +import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages'; +import { datasource } from './common'; + +describe('datasource/gitlab-packages/index', () => { + describe('getReleases', () => { + it('returns package from custom registry', async () => { + const body = [ + { + version: '1.0.0', + created_at: '2020-03-04T12:01:37.000-06:00', + name: 'mypkg', + }, + { + version: 'v1.1.0', + created_at: '2020-04-04T12:01:37.000-06:00', + name: 'mypkg', + }, + { + version: 'v1.1.1', + created_at: '2020-05-04T12:01:37.000-06:00', + name: 'mypkg', + }, + { + version: 'v2.0.0', + created_at: '2020-05-04T12:01:37.000-06:00', + name: 'otherpkg', + }, + ]; + httpMock + .scope('https://gitlab.com') + .get('/api/v4/projects/user%2Fproject1/packages') + .query({ + package_name: 'mypkg', + per_page: '100', + }) + .reply(200, body); + const res = await getPkgReleases({ + datasource, + registryUrls: ['https://gitlab.com'], + depName: 'user/project1:mypkg', + }); + expect(res).toMatchSnapshot(); + expect(res.releases).toHaveLength(3); + }); + + it('returns null for 404', async () => { + httpMock + .scope('https://gitlab.com') + .get('/api/v4/projects/user%2Fproject1/packages') + .query({ + package_name: 'mypkg', + per_page: '100', + }) + .reply(404); + expect( + await getPkgReleases({ + datasource, + registryUrls: ['https://gitlab.com'], + depName: 'user/project1:mypkg', + }) + ).toBeNull(); + }); + + it('returns null for empty 200 OK', async () => { + httpMock + .scope('https://gitlab.com') + .get('/api/v4/projects/user%2Fproject1/packages') + .query({ + package_name: 'mypkg', + per_page: '100', + }) + .reply(200, []); + expect( + await getPkgReleases({ + datasource, + registryUrls: ['https://gitlab.com'], + depName: 'user/project1:mypkg', + }) + ).toBeNull(); + }); + + it('throws for 5xx', async () => { + httpMock + .scope('https://gitlab.com') + .get('/api/v4/projects/user%2Fproject1/packages') + .query({ + package_name: 'mypkg', + per_page: '100', + }) + .reply(502); + await expect( + getPkgReleases({ + datasource, + registryUrls: ['https://gitlab.com'], + depName: 'user/project1:mypkg', + }) + ).rejects.toThrow(EXTERNAL_HOST_ERROR); + }); + }); +}); diff --git a/lib/datasource/gitlab-packages/index.ts b/lib/datasource/gitlab-packages/index.ts new file mode 100644 index 00000000000000..2d36c15733b079 --- /dev/null +++ b/lib/datasource/gitlab-packages/index.ts @@ -0,0 +1,84 @@ +import { cache } from '../../util/cache/package/decorator'; +import { GitlabHttp } from '../../util/http/gitlab'; +import { joinUrlParts } from '../../util/url'; +import { Datasource } from '../datasource'; +import type { GetReleasesConfig, ReleaseResult } from '../types'; +import { datasource } from './common'; +import type { GitlabPackage } from './types'; + +// Gitlab Packages API: https://docs.gitlab.com/ee/api/packages.html + +export class GitlabPackagesDatasource extends Datasource { + static readonly id = datasource; + + protected override http: GitlabHttp; + + override caching = true; + + override customRegistrySupport = true; + + override defaultRegistryUrls = ['https://gitlab.com']; + + constructor() { + super(datasource); + this.http = new GitlabHttp(); + } + + static getGitlabPackageApiUrl( + registryUrl: string, + projectName: string, + packageName: string + ): string { + const projectNameEncoded = encodeURIComponent(projectName); + const packageNameEncoded = encodeURIComponent(packageName); + + return joinUrlParts( + registryUrl, + `api/v4/projects`, + projectNameEncoded, + `packages?package_name=${packageNameEncoded}&per_page=100` + ); + } + + @cache({ + namespace: `datasource-${datasource}`, + key: ({ registryUrl, lookupName }: GetReleasesConfig) => + `${registryUrl}-${lookupName}`, + }) + async getReleases({ + registryUrl, + lookupName, + }: GetReleasesConfig): Promise { + const [projectName, packageName] = lookupName.split(':', 2); + + const apiUrl = GitlabPackagesDatasource.getGitlabPackageApiUrl( + registryUrl, + projectName, + packageName + ); + + const result: ReleaseResult = { + releases: null, + }; + + let response: GitlabPackage[]; + try { + response = ( + await this.http.getJson(apiUrl, { paginate: true }) + ).body; + + result.releases = response + // Setting the package_name option when calling the GitLab API isn't enough to filter information about other packages + // because this option is only implemented on GitLab > 12.9 and it only does a fuzzy search. + .filter((r) => r.name === packageName) + .map(({ version, created_at }) => ({ + version, + releaseTimestamp: created_at, + })); + } catch (err) { + this.handleGenericErrors(err); + } + + return result.releases?.length ? result : null; + } +} diff --git a/lib/datasource/gitlab-packages/readme.md b/lib/datasource/gitlab-packages/readme.md new file mode 100644 index 00000000000000..369f7e9ffc61ae --- /dev/null +++ b/lib/datasource/gitlab-packages/readme.md @@ -0,0 +1,39 @@ +[GitLab Packages API](https://docs.gitlab.com/ee/api/packages.html) supports looking up package versions from [all types of packages registry supported by GitLab](https://docs.gitlab.com/ee/user/packages/package_registry/index.html) and can be used in combination with [regex managers](https://docs.renovatebot.com/modules/manager/regex/) to keep dependencies up-to-date which are not specifically supported by Renovate. + +To specify which specific repository should be queried when looking up a package, the `depName` should be formed with the project path first, then a `:` and finally the package name. + +As an example, `gitlab-org/ci-cd/package-stage/feature-testing/new-packages-list:@gitlab-org/nk-js` would look for the`@gitlab-org/nk-js` packages in the generic packages repository of the `gitlab-org/ci-cd/package-stage/feature-testing/new-packages-list` project. + +If you are using a self-hosted GitLab instance, please note the following requirements: + +- If you are on the `Free` edition, this datasource requires at least GitLab 13.3 +- If you are on the `Premium` or the `Ultimate` edition, this datasource requires at least GitLab 11.8, but GitLab 12.9 or more is recommended if you have a lot of packages with different names in the same project + +**Usage Example** + +A real-world example for this specific datasource would be maintaining package versions in a config file. +This can be achieved by configuring a generic regex manager in `renovate.json` for files named `versions.ini`: + +```json +{ + "regexManagers": [ + { + "fileMatch": ["^versions.ini$"], + "matchStrings": [ + "# renovate: datasource=(?.*?) depName=(?.*?)( versioning=(?.*?))?( registryUrl=(?.*?))?\\s.*?_VERSION=(?.*)\\s" + ], + "versioningTemplate": "{{#if versioning}}{{{versioning}}}{{else}}semver{{/if}}" + } + ] +} +``` + +Now you may use comments in your `versions.ini` files to automatically update dependencies, which could look like this: + +```ini +# renovate: datasource=gitlab-packages depName=gitlab-org/ci-cd/package-stage/feature-testing/new-packages-list:@gitlab-org/nk-js versioning=semver registryUrl=https://gitlab.com +NKJS_VERSION=3.4.0 + +``` + +By default, `gitlab-packages` uses the `docker` versioning scheme. diff --git a/lib/datasource/gitlab-packages/types.ts b/lib/datasource/gitlab-packages/types.ts new file mode 100644 index 00000000000000..e68eecfa1fd153 --- /dev/null +++ b/lib/datasource/gitlab-packages/types.ts @@ -0,0 +1,5 @@ +export interface GitlabPackage { + version: string; + created_at: string; + name: string; +}