feat(datasource): add Gitlab Package support (#11672)

Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com>
Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
Co-authored-by: Rhys Arkins <rhys@arkins.net>

lib/constants/platform.spec.ts

@@ -1,5 +1,6 @@
 import { id as GH_RELEASES_DS } from '../datasource/github-releases';
 import { id as GH_TAGS_DS } from '../datasource/github-tags';
+import { GitlabPackagesDatasource } from '../datasource/gitlab-packages';
 import { GitlabReleasesDatasource } from '../datasource/gitlab-releases';
 import { id as GL_TAGS_DS } from '../datasource/gitlab-tags';
 import { id as POD_DS } from '../datasource/pod';
@@ -16,6 +17,9 @@ describe('constants/platform', () => {
     expect(
       GITLAB_API_USING_HOST_TYPES.includes(GitlabReleasesDatasource.id)
     ).toBeTrue();
+    expect(
+      GITLAB_API_USING_HOST_TYPES.includes(GitlabPackagesDatasource.id)
+    ).toBeTrue();
     expect(
       GITLAB_API_USING_HOST_TYPES.includes(PLATFORM_TYPE_GITLAB)
     ).toBeTrue();

lib/constants/platforms.ts

@@ -16,4 +16,5 @@ export const GITLAB_API_USING_HOST_TYPES = [
   PLATFORM_TYPE_GITLAB,
   'gitlab-releases',
   'gitlab-tags',
+  'gitlab-packages',
 ];

lib/datasource/api.ts

@@ -11,6 +11,7 @@ import * as gitRefs from './git-refs';
 import * as gitTags from './git-tags';
 import * as githubReleases from './github-releases';
 import * as githubTags from './github-tags';
+import { GitlabPackagesDatasource } from './gitlab-packages';
 import { GitlabReleasesDatasource } from './gitlab-releases';
 import * as gitlabTags from './gitlab-tags';
 import * as go from './go';
@@ -51,6 +52,7 @@ api.set('git-refs', gitRefs);
 api.set('git-tags', gitTags);
 api.set('github-releases', githubReleases);
 api.set('github-tags', githubTags);
+api.set('gitlab-packages', new GitlabPackagesDatasource());
 api.set('gitlab-tags', gitlabTags);
 api.set(GitlabReleasesDatasource.id, new GitlabReleasesDatasource());
 api.set('go', go);

lib/datasource/gitlab-packages/__snapshots__/index.spec.ts.snap

@@ -0,0 +1,21 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`datasource/gitlab-packages/index getReleases returns package from custom registry 1`] = `
+Object {
+  "registryUrl": "https://gitlab.com",
+  "releases": Array [
+    Object {
+      "releaseTimestamp": "2020-03-04T18:01:37.000Z",
+      "version": "1.0.0",
+    },
+    Object {
+      "releaseTimestamp": "2020-04-04T18:01:37.000Z",
+      "version": "v1.1.0",
+    },
+    Object {
+      "releaseTimestamp": "2020-05-04T18:01:37.000Z",
+      "version": "v1.1.1",
+    },
+  ],
+}
+`;

lib/datasource/gitlab-packages/common.ts

@@ -0,0 +1 @@
+export const datasource = 'gitlab-packages';

lib/datasource/gitlab-packages/index.spec.ts

@@ -0,0 +1,102 @@
+import { getPkgReleases } from '..';
+import * as httpMock from '../../../test/http-mock';
+import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages';
+import { datasource } from './common';
+
+describe('datasource/gitlab-packages/index', () => {
+  describe('getReleases', () => {
+    it('returns package from custom registry', async () => {
+      const body = [
+        {
+          version: '1.0.0',
+          created_at: '2020-03-04T12:01:37.000-06:00',
+          name: 'mypkg',
+        },
+        {
+          version: 'v1.1.0',
+          created_at: '2020-04-04T12:01:37.000-06:00',
+          name: 'mypkg',
+        },
+        {
+          version: 'v1.1.1',
+          created_at: '2020-05-04T12:01:37.000-06:00',
+          name: 'mypkg',
+        },
+        {
+          version: 'v2.0.0',
+          created_at: '2020-05-04T12:01:37.000-06:00',
+          name: 'otherpkg',
+        },
+      ];
+      httpMock
+        .scope('https://gitlab.com')
+        .get('/api/v4/projects/user%2Fproject1/packages')
+        .query({
+          package_name: 'mypkg',
+          per_page: '100',
+        })
+        .reply(200, body);
+      const res = await getPkgReleases({
+        datasource,
+        registryUrls: ['https://gitlab.com'],
+        depName: 'user/project1:mypkg',
+      });
+      expect(res).toMatchSnapshot();
+      expect(res.releases).toHaveLength(3);
+    });
+
+    it('returns null for 404', async () => {
+      httpMock
+        .scope('https://gitlab.com')
+        .get('/api/v4/projects/user%2Fproject1/packages')
+        .query({
+          package_name: 'mypkg',
+          per_page: '100',
+        })
+        .reply(404);
+      expect(
+        await getPkgReleases({
+          datasource,
+          registryUrls: ['https://gitlab.com'],
+          depName: 'user/project1:mypkg',
+        })
+      ).toBeNull();
+    });
+
+    it('returns null for empty 200 OK', async () => {
+      httpMock
+        .scope('https://gitlab.com')
+        .get('/api/v4/projects/user%2Fproject1/packages')
+        .query({
+          package_name: 'mypkg',
+          per_page: '100',
+        })
+        .reply(200, []);
+      expect(
+        await getPkgReleases({
+          datasource,
+          registryUrls: ['https://gitlab.com'],
+          depName: 'user/project1:mypkg',
+        })
+      ).toBeNull();
+    });
+
+    it('throws for 5xx', async () => {
+      httpMock
+        .scope('https://gitlab.com')
+        .get('/api/v4/projects/user%2Fproject1/packages')
+        .query({
+          package_name: 'mypkg',
+          per_page: '100',
+        })
+        .reply(502);
+      await expect(
+        getPkgReleases({
+          datasource,
+          registryUrls: ['https://gitlab.com'],
+          depName: 'user/project1:mypkg',
+        })
+      ).rejects.toThrow(EXTERNAL_HOST_ERROR);
+    });
+  });
+});

lib/datasource/gitlab-packages/index.ts

@@ -0,0 +1,84 @@
+import { cache } from '../../util/cache/package/decorator';
+import { GitlabHttp } from '../../util/http/gitlab';
+import { joinUrlParts } from '../../util/url';
+import { Datasource } from '../datasource';
+import type { GetReleasesConfig, ReleaseResult } from '../types';
+import { datasource } from './common';
+import type { GitlabPackage } from './types';
+
+// Gitlab Packages API: https://docs.gitlab.com/ee/api/packages.html
+
+export class GitlabPackagesDatasource extends Datasource {
+  static readonly id = datasource;
+
+  protected override http: GitlabHttp;
+
+  override caching = true;
+
+  override customRegistrySupport = true;
+
+  override defaultRegistryUrls = ['https://gitlab.com'];
+
+  constructor() {
+    super(datasource);
+    this.http = new GitlabHttp();
+  }
+
+  static getGitlabPackageApiUrl(
+    registryUrl: string,
+    projectName: string,
+    packageName: string
+  ): string {
+    const projectNameEncoded = encodeURIComponent(projectName);
+    const packageNameEncoded = encodeURIComponent(packageName);
+
+    return joinUrlParts(
+      registryUrl,
+      `api/v4/projects`,
+      projectNameEncoded,
+      `packages?package_name=${packageNameEncoded}&per_page=100`
+    );
+  }
+
+  @cache({
+    namespace: `datasource-${datasource}`,
+    key: ({ registryUrl, lookupName }: GetReleasesConfig) =>
+      `${registryUrl}-${lookupName}`,
+  })
+  async getReleases({
+    registryUrl,
+    lookupName,
+  }: GetReleasesConfig): Promise<ReleaseResult | null> {
+    const [projectName, packageName] = lookupName.split(':', 2);
+
+    const apiUrl = GitlabPackagesDatasource.getGitlabPackageApiUrl(
+      registryUrl,
+      projectName,
+      packageName
+    );
+
+    const result: ReleaseResult = {
+      releases: null,
+    };
+
+    let response: GitlabPackage[];
+    try {
+      response = (
+        await this.http.getJson<GitlabPackage[]>(apiUrl, { paginate: true })
+      ).body;
+
+      result.releases = response
+        // Setting the package_name option when calling the GitLab API isn't enough to filter information about other packages
+        // because this option is only implemented on GitLab > 12.9 and it only does a fuzzy search.
+        .filter((r) => r.name === packageName)
+        .map(({ version, created_at }) => ({
+          version,
+          releaseTimestamp: created_at,
+        }));
+    } catch (err) {
+      this.handleGenericErrors(err);
+    }
+
+    return result.releases?.length ? result : null;
+  }
+}

lib/datasource/gitlab-packages/readme.md

@@ -0,0 +1,39 @@
+[GitLab Packages API](https://docs.gitlab.com/ee/api/packages.html) supports looking up package versions from [all types of packages registry supported by GitLab](https://docs.gitlab.com/ee/user/packages/package_registry/index.html) and can be used in combination with [regex managers](https://docs.renovatebot.com/modules/manager/regex/) to keep dependencies up-to-date which are not specifically supported by Renovate.
+
+To specify which specific repository should be queried when looking up a package, the `depName` should be formed with the project path first, then a `:` and finally the package name.
+
+As an example, `gitlab-org/ci-cd/package-stage/feature-testing/new-packages-list:@gitlab-org/nk-js` would look for the`@gitlab-org/nk-js` packages in the generic packages repository of the `gitlab-org/ci-cd/package-stage/feature-testing/new-packages-list` project.
+
+If you are using a self-hosted GitLab instance, please note the following requirements:
+
+- If you are on the `Free` edition, this datasource requires at least GitLab 13.3
+- If you are on the `Premium` or the `Ultimate` edition, this datasource requires at least GitLab 11.8, but GitLab 12.9 or more is recommended if you have a lot of packages with different names in the same project
+
+**Usage Example**
+
+A real-world example for this specific datasource would be maintaining package versions in a config file.
+This can be achieved by configuring a generic regex manager in `renovate.json` for files named `versions.ini`:
+
+```json
+{
+  "regexManagers": [
+    {
+      "fileMatch": ["^versions.ini$"],
+      "matchStrings": [
+        "# renovate: datasource=(?<datasource>.*?) depName=(?<depName>.*?)( versioning=(?<versioning>.*?))?( registryUrl=(?<registryUrl>.*?))?\\s.*?_VERSION=(?<currentValue>.*)\\s"
+      ],
+      "versioningTemplate": "{{#if versioning}}{{{versioning}}}{{else}}semver{{/if}}"
+    }
+  ]
+}
+```
+
+Now you may use comments in your `versions.ini` files to automatically update dependencies, which could look like this:
+
+```ini
+# renovate: datasource=gitlab-packages depName=gitlab-org/ci-cd/package-stage/feature-testing/new-packages-list:@gitlab-org/nk-js versioning=semver registryUrl=https://gitlab.com
+NKJS_VERSION=3.4.0
+
+```
+
+By default, `gitlab-packages` uses the `docker` versioning scheme.

lib/datasource/gitlab-packages/types.ts

@@ -0,0 +1,5 @@
+export interface GitlabPackage {
+  version: string;
+  created_at: string;
+  name: string;
+}