From 3d8e3ad12dc7b4903307d6ce3993b7b8a0300e3d Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Fri, 19 Jun 2020 21:29:34 +0200 Subject: [PATCH] feat(internal): datasource registryStrategy (#6549) --- .../__snapshots__/index.spec.ts.snap | 15 + lib/datasource/common.ts | 7 +- lib/datasource/docker/index.spec.ts | 7 +- lib/datasource/docker/index.ts | 23 +- lib/datasource/gitlab-tags/index.ts | 11 +- .../__snapshots__/index.spec.ts.snap | 264 ------------------ lib/datasource/gradle-version/index.ts | 67 ++--- lib/datasource/helm/index.ts | 4 +- lib/datasource/index.spec.ts | 102 +++++++ lib/datasource/index.ts | 134 ++++++++- lib/datasource/maven/index.ts | 51 ++-- lib/datasource/nuget/index.ts | 47 +--- lib/datasource/packagist/index.ts | 18 +- lib/datasource/pod/index.ts | 38 ++- .../pypi/__snapshots__/index.spec.ts.snap | 15 - lib/datasource/pypi/index.spec.ts | 14 - lib/datasource/pypi/index.ts | 44 +-- lib/datasource/rubygems/index.ts | 1 + lib/datasource/rubygems/releases.ts | 19 +- lib/datasource/sbt-package/index.ts | 13 +- lib/datasource/sbt-plugin/index.ts | 13 +- lib/datasource/terraform-module/index.ts | 13 +- lib/datasource/terraform-provider/index.ts | 1 - 23 files changed, 386 insertions(+), 535 deletions(-) diff --git a/lib/datasource/__snapshots__/index.spec.ts.snap b/lib/datasource/__snapshots__/index.spec.ts.snap index 269e3ed3267dde..d5dffbcb638235 100644 --- a/lib/datasource/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/__snapshots__/index.spec.ts.snap @@ -22,3 +22,18 @@ Object { "sourceUrl": "https://github.com/nodejs/node", } `; + +exports[`datasource/index merges registries and returns success 1`] = ` +Object { + "releases": Array [ + Object { + "version": "1.0.0", + }, + Object { + "version": "1.1.0", + }, + ], +} +`; + +exports[`datasource/index warns if multiple registryUrls for registryStrategy=first 1`] = `null`; diff --git a/lib/datasource/common.ts b/lib/datasource/common.ts index d9c0ce47123fdd..1d4be8b8c4d349 100644 --- a/lib/datasource/common.ts +++ b/lib/datasource/common.ts @@ -7,7 +7,9 @@ export interface Config { registryUrls?: string[]; } -export type DigestConfig = Config; +export interface DigestConfig extends Config { + registryUrl?: string; +} interface ReleasesConfigBase { compatibility?: Record; @@ -17,6 +19,7 @@ interface ReleasesConfigBase { export interface GetReleasesConfig extends ReleasesConfigBase { lookupName: string; + registryUrl?: string; } export interface GetPkgReleasesConfig extends ReleasesConfigBase { @@ -65,6 +68,7 @@ export interface ReleaseResult { sourceUrl?: string; tags?: Record; versions?: any; + registryUrl?: string; } export interface Datasource { @@ -74,6 +78,7 @@ export interface Datasource { defaultRegistryUrls?: string[]; appendRegistryUrls?: string[]; defaultConfig?: object; + registryStrategy?: 'first' | 'hunt' | 'merge'; } export class DatasourceError extends Error { diff --git a/lib/datasource/docker/index.spec.ts b/lib/datasource/docker/index.spec.ts index f45a42b8b7efcb..7d2c09c7ffe7d4 100644 --- a/lib/datasource/docker/index.spec.ts +++ b/lib/datasource/docker/index.spec.ts @@ -31,13 +31,16 @@ describe('api/docker', () => { describe('getRegistryRepository', () => { it('handles local registries', () => { - const res = docker.getRegistryRepository('registry:5000/org/package', []); + const res = docker.getRegistryRepository( + 'registry:5000/org/package', + 'https://index.docker.io' + ); expect(res).toMatchSnapshot(); }); it('supports registryUrls', () => { const res = docker.getRegistryRepository( 'my.local.registry/prefix/image', - ['https://my.local.registry/prefix'] + 'https://my.local.registry/prefix' ); expect(res).toMatchSnapshot(); }); diff --git a/lib/datasource/docker/index.ts b/lib/datasource/docker/index.ts index f90cb7bb662c95..34536479d3370f 100644 --- a/lib/datasource/docker/index.ts +++ b/lib/datasource/docker/index.ts @@ -1,6 +1,5 @@ import { OutgoingHttpHeaders } from 'http'; import URL from 'url'; -import is from '@sindresorhus/is'; import AWS from 'aws-sdk'; import hasha from 'hasha'; import parseLinkHeader from 'parse-link-header'; @@ -16,6 +15,8 @@ import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common'; // TODO: replace www-authenticate with https://www.npmjs.com/package/auth-header ? export const id = 'docker'; +export const defaultRegistryUrls = ['https://index.docker.io']; +export const registryStrategy = 'first'; export const defaultConfig = { managerBranchPrefix: 'docker-', @@ -57,10 +58,10 @@ export interface RegistryRepository { export function getRegistryRepository( lookupName: string, - registryUrls: string[] + registryUrl: string ): RegistryRepository { - if (is.nonEmptyArray(registryUrls)) { - const dockerRegistry = registryUrls[0] + if (registryUrl !== defaultRegistryUrls[0]) { + const dockerRegistry = registryUrl .replace('https://', '') .replace(/\/?$/, '/'); if (lookupName.startsWith(dockerRegistry)) { @@ -77,10 +78,10 @@ export function getRegistryRepository( split.shift(); } let repository = split.join('/'); - if (!registry && is.nonEmptyArray(registryUrls)) { - [registry] = registryUrls; + if (!registry) { + registry = registryUrl; } - if (!registry || registry === 'docker.io') { + if (registry === 'docker.io') { registry = 'index.docker.io'; } if (!/^https?:\/\//.exec(registry)) { @@ -327,12 +328,12 @@ async function getManifestResponse( * - Return the digest as a string */ export async function getDigest( - { registryUrls, lookupName }: GetReleasesConfig, + { registryUrl, lookupName }: GetReleasesConfig, newValue?: string ): Promise { const { registry, repository } = getRegistryRepository( lookupName, - registryUrls + registryUrl ); logger.debug(`getDigest(${registry}, ${repository}, ${newValue})`); const newTag = newValue || 'latest'; @@ -608,11 +609,11 @@ async function getLabels( */ export async function getReleases({ lookupName, - registryUrls, + registryUrl, }: GetReleasesConfig): Promise { const { registry, repository } = getRegistryRepository( lookupName, - registryUrls + registryUrl ); const tags = await getTags(registry, repository); if (!tags) { diff --git a/lib/datasource/gitlab-tags/index.ts b/lib/datasource/gitlab-tags/index.ts index 0ddd79e73c679a..f9c795ce24ed6b 100644 --- a/lib/datasource/gitlab-tags/index.ts +++ b/lib/datasource/gitlab-tags/index.ts @@ -1,5 +1,4 @@ import URL from 'url'; -import is from '@sindresorhus/is'; import { logger } from '../../logger'; import * as globalCache from '../../util/cache/global'; import { GitlabHttp } from '../../util/http/gitlab'; @@ -8,6 +7,8 @@ import { GetReleasesConfig, ReleaseResult } from '../common'; const gitlabApi = new GitlabHttp(); export const id = 'gitlab-tags'; +export const defaultRegistryUrls = ['https://gitlab.com']; +export const registryStrategy = 'first'; const cacheNamespace = 'datasource-gitlab'; function getCacheKey(depHost: string, repo: string): string { @@ -23,13 +24,9 @@ type GitlabTag = { }; export async function getReleases({ - registryUrls, + registryUrl: depHost, lookupName: repo, }: GetReleasesConfig): Promise { - // Use registryUrls if present, otherwise default to publid gitlab.com - const depHost = is.nonEmptyArray(registryUrls) - ? registryUrls[0].replace(/\/$/, '') - : 'https://gitlab.com'; let gitlabTags: GitlabTag[]; const cachedResult = await globalCache.get( cacheNamespace, @@ -65,7 +62,7 @@ export async function getReleases({ } const dependency: ReleaseResult = { - sourceUrl: `${depHost}/${repo}`, + sourceUrl: URL.resolve(depHost, repo), releases: null, }; dependency.releases = gitlabTags.map(({ name, commit }) => ({ diff --git a/lib/datasource/gradle-version/__snapshots__/index.spec.ts.snap b/lib/datasource/gradle-version/__snapshots__/index.spec.ts.snap index f2dc398b8076f7..3bb0a0d4ffa8c1 100644 --- a/lib/datasource/gradle-version/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/gradle-version/__snapshots__/index.spec.ts.snap @@ -4,10 +4,6 @@ exports[`datasource/gradle-version getReleases calls configured registryUrls 1`] Object { "homepage": "https://gradle.org", "releases": Array [ - Object { - "releaseTimestamp": "2009-07-20T08:50:13+0200", - "version": "0.7", - }, Object { "releaseTimestamp": "2009-07-20T08:50:13+0200", "version": "0.7", @@ -16,22 +12,10 @@ Object { "releaseTimestamp": "2009-09-28T14:01:59+0200", "version": "0.8", }, - Object { - "releaseTimestamp": "2009-09-28T14:01:59+0200", - "version": "0.8", - }, Object { "releaseTimestamp": "2010-12-19T12:50:06+1100", "version": "0.9", }, - Object { - "releaseTimestamp": "2010-12-19T12:50:06+1100", - "version": "0.9", - }, - Object { - "releaseTimestamp": "2011-01-02T11:40:57+1100", - "version": "0.9.1", - }, Object { "releaseTimestamp": "2011-01-02T11:40:57+1100", "version": "0.9.1", @@ -40,14 +24,6 @@ Object { "releaseTimestamp": "2011-01-23T13:34:21+1100", "version": "0.9.2", }, - Object { - "releaseTimestamp": "2011-01-23T13:34:21+1100", - "version": "0.9.2", - }, - Object { - "releaseTimestamp": "2012-06-12T02:56:21+0200", - "version": "1.0", - }, Object { "releaseTimestamp": "2012-06-12T02:56:21+0200", "version": "1.0", @@ -56,14 +32,6 @@ Object { "releaseTimestamp": "2012-07-31T13:24:32+0000", "version": "1.1", }, - Object { - "releaseTimestamp": "2012-07-31T13:24:32+0000", - "version": "1.1", - }, - Object { - "releaseTimestamp": "2012-09-12T10:46:02+0000", - "version": "1.2", - }, Object { "releaseTimestamp": "2012-09-12T10:46:02+0000", "version": "1.2", @@ -72,14 +40,6 @@ Object { "releaseTimestamp": "2012-11-20T11:37:38+0000", "version": "1.3", }, - Object { - "releaseTimestamp": "2012-11-20T11:37:38+0000", - "version": "1.3", - }, - Object { - "releaseTimestamp": "2013-01-28T03:42:46+0000", - "version": "1.4", - }, Object { "releaseTimestamp": "2013-01-28T03:42:46+0000", "version": "1.4", @@ -88,14 +48,6 @@ Object { "releaseTimestamp": "2013-03-27T14:09:35+0000", "version": "1.5", }, - Object { - "releaseTimestamp": "2013-03-27T14:09:35+0000", - "version": "1.5", - }, - Object { - "releaseTimestamp": "2013-05-07T09:12:14+0000", - "version": "1.6", - }, Object { "releaseTimestamp": "2013-05-07T09:12:14+0000", "version": "1.6", @@ -104,14 +56,6 @@ Object { "releaseTimestamp": "2013-08-06T11:19:56+0000", "version": "1.7", }, - Object { - "releaseTimestamp": "2013-08-06T11:19:56+0000", - "version": "1.7", - }, - Object { - "releaseTimestamp": "2013-09-24T07:32:33+0000", - "version": "1.8", - }, Object { "releaseTimestamp": "2013-09-24T07:32:33+0000", "version": "1.8", @@ -120,14 +64,6 @@ Object { "releaseTimestamp": "2013-11-19T08:20:02+0000", "version": "1.9", }, - Object { - "releaseTimestamp": "2013-11-19T08:20:02+0000", - "version": "1.9", - }, - Object { - "releaseTimestamp": "2013-12-17T09:28:15+0000", - "version": "1.10", - }, Object { "releaseTimestamp": "2013-12-17T09:28:15+0000", "version": "1.10", @@ -136,14 +72,6 @@ Object { "releaseTimestamp": "2014-02-11T11:34:39+0000", "version": "1.11", }, - Object { - "releaseTimestamp": "2014-02-11T11:34:39+0000", - "version": "1.11", - }, - Object { - "releaseTimestamp": "2014-04-29T09:24:31+0000", - "version": "1.12", - }, Object { "releaseTimestamp": "2014-04-29T09:24:31+0000", "version": "1.12", @@ -152,14 +80,6 @@ Object { "releaseTimestamp": "2014-07-01T07:45:34+0000", "version": "2.0", }, - Object { - "releaseTimestamp": "2014-07-01T07:45:34+0000", - "version": "2.0", - }, - Object { - "releaseTimestamp": "2014-09-08T10:40:39+0000", - "version": "2.1", - }, Object { "releaseTimestamp": "2014-09-08T10:40:39+0000", "version": "2.1", @@ -168,14 +88,6 @@ Object { "releaseTimestamp": "2014-11-10T13:31:44+0000", "version": "2.2", }, - Object { - "releaseTimestamp": "2014-11-10T13:31:44+0000", - "version": "2.2", - }, - Object { - "releaseTimestamp": "2014-11-24T09:45:35+0000", - "version": "2.2.1", - }, Object { "releaseTimestamp": "2014-11-24T09:45:35+0000", "version": "2.2.1", @@ -184,14 +96,6 @@ Object { "releaseTimestamp": "2015-02-16T05:09:33+0000", "version": "2.3", }, - Object { - "releaseTimestamp": "2015-02-16T05:09:33+0000", - "version": "2.3", - }, - Object { - "releaseTimestamp": "2015-05-05T08:09:24+0000", - "version": "2.4", - }, Object { "releaseTimestamp": "2015-05-05T08:09:24+0000", "version": "2.4", @@ -200,14 +104,6 @@ Object { "releaseTimestamp": "2015-07-08T07:38:37+0000", "version": "2.5", }, - Object { - "releaseTimestamp": "2015-07-08T07:38:37+0000", - "version": "2.5", - }, - Object { - "releaseTimestamp": "2015-08-10T13:15:06+0000", - "version": "2.6", - }, Object { "releaseTimestamp": "2015-08-10T13:15:06+0000", "version": "2.6", @@ -216,22 +112,10 @@ Object { "releaseTimestamp": "2015-09-14T07:26:16+0000", "version": "2.7", }, - Object { - "releaseTimestamp": "2015-09-14T07:26:16+0000", - "version": "2.7", - }, Object { "releaseTimestamp": "2015-10-20T03:46:36+0000", "version": "2.8", }, - Object { - "releaseTimestamp": "2015-10-20T03:46:36+0000", - "version": "2.8", - }, - Object { - "releaseTimestamp": "2015-11-17T07:02:17+0000", - "version": "2.9", - }, Object { "releaseTimestamp": "2015-11-17T07:02:17+0000", "version": "2.9", @@ -240,14 +124,6 @@ Object { "releaseTimestamp": "2015-12-21T21:15:04+0000", "version": "2.10", }, - Object { - "releaseTimestamp": "2015-12-21T21:15:04+0000", - "version": "2.10", - }, - Object { - "releaseTimestamp": "2016-02-08T07:59:16+0000", - "version": "2.11", - }, Object { "releaseTimestamp": "2016-02-08T07:59:16+0000", "version": "2.11", @@ -256,14 +132,6 @@ Object { "releaseTimestamp": "2016-03-14T08:32:03+0000", "version": "2.12", }, - Object { - "releaseTimestamp": "2016-03-14T08:32:03+0000", - "version": "2.12", - }, - Object { - "releaseTimestamp": "2016-04-25T04:10:10+0000", - "version": "2.13", - }, Object { "releaseTimestamp": "2016-04-25T04:10:10+0000", "version": "2.13", @@ -272,14 +140,6 @@ Object { "releaseTimestamp": "2016-06-14T07:16:37+0000", "version": "2.14", }, - Object { - "releaseTimestamp": "2016-06-14T07:16:37+0000", - "version": "2.14", - }, - Object { - "releaseTimestamp": "2016-07-18T06:38:37+0000", - "version": "2.14.1", - }, Object { "releaseTimestamp": "2016-07-18T06:38:37+0000", "version": "2.14.1", @@ -288,14 +148,6 @@ Object { "releaseTimestamp": "2016-08-15T13:15:01+0000", "version": "3.0", }, - Object { - "releaseTimestamp": "2016-08-15T13:15:01+0000", - "version": "3.0", - }, - Object { - "releaseTimestamp": "2016-09-19T10:53:53+0000", - "version": "3.1", - }, Object { "releaseTimestamp": "2016-09-19T10:53:53+0000", "version": "3.1", @@ -304,14 +156,6 @@ Object { "releaseTimestamp": "2016-11-14T12:32:59+0000", "version": "3.2", }, - Object { - "releaseTimestamp": "2016-11-14T12:32:59+0000", - "version": "3.2", - }, - Object { - "releaseTimestamp": "2016-11-22T15:19:54+0000", - "version": "3.2.1", - }, Object { "releaseTimestamp": "2016-11-22T15:19:54+0000", "version": "3.2.1", @@ -320,14 +164,6 @@ Object { "releaseTimestamp": "2017-01-03T15:31:04+0000", "version": "3.3", }, - Object { - "releaseTimestamp": "2017-01-03T15:31:04+0000", - "version": "3.3", - }, - Object { - "releaseTimestamp": "2017-02-20T14:49:26+0000", - "version": "3.4", - }, Object { "releaseTimestamp": "2017-02-20T14:49:26+0000", "version": "3.4", @@ -336,14 +172,6 @@ Object { "releaseTimestamp": "2017-03-03T19:45:41+0000", "version": "3.4.1", }, - Object { - "releaseTimestamp": "2017-03-03T19:45:41+0000", - "version": "3.4.1", - }, - Object { - "releaseTimestamp": "2017-04-10T13:37:25+0000", - "version": "3.5", - }, Object { "releaseTimestamp": "2017-04-10T13:37:25+0000", "version": "3.5", @@ -352,14 +180,6 @@ Object { "releaseTimestamp": "2017-06-16T14:36:27+0000", "version": "3.5.1", }, - Object { - "releaseTimestamp": "2017-06-16T14:36:27+0000", - "version": "3.5.1", - }, - Object { - "releaseTimestamp": "2017-06-14T15:11:08+0000", - "version": "4.0", - }, Object { "releaseTimestamp": "2017-06-14T15:11:08+0000", "version": "4.0", @@ -368,14 +188,6 @@ Object { "releaseTimestamp": "2017-07-07T14:02:41+0000", "version": "4.0.1", }, - Object { - "releaseTimestamp": "2017-07-07T14:02:41+0000", - "version": "4.0.1", - }, - Object { - "releaseTimestamp": "2017-07-26T16:19:18+0000", - "version": "4.0.2", - }, Object { "releaseTimestamp": "2017-07-26T16:19:18+0000", "version": "4.0.2", @@ -384,14 +196,6 @@ Object { "releaseTimestamp": "2017-08-07T14:38:48+0000", "version": "4.1", }, - Object { - "releaseTimestamp": "2017-08-07T14:38:48+0000", - "version": "4.1", - }, - Object { - "releaseTimestamp": "2017-09-20T14:48:23+0000", - "version": "4.2", - }, Object { "releaseTimestamp": "2017-09-20T14:48:23+0000", "version": "4.2", @@ -400,14 +204,6 @@ Object { "releaseTimestamp": "2017-10-02T15:36:21+0000", "version": "4.2.1", }, - Object { - "releaseTimestamp": "2017-10-02T15:36:21+0000", - "version": "4.2.1", - }, - Object { - "releaseTimestamp": "2017-10-30T15:43:29+0000", - "version": "4.3", - }, Object { "releaseTimestamp": "2017-10-30T15:43:29+0000", "version": "4.3", @@ -416,14 +212,6 @@ Object { "releaseTimestamp": "2017-11-08T08:59:45+0000", "version": "4.3.1", }, - Object { - "releaseTimestamp": "2017-11-08T08:59:45+0000", - "version": "4.3.1", - }, - Object { - "releaseTimestamp": "2017-12-06T09:05:06+0000", - "version": "4.4", - }, Object { "releaseTimestamp": "2017-12-06T09:05:06+0000", "version": "4.4", @@ -432,14 +220,6 @@ Object { "releaseTimestamp": "2017-12-20T15:45:23+0000", "version": "4.4.1", }, - Object { - "releaseTimestamp": "2017-12-20T15:45:23+0000", - "version": "4.4.1", - }, - Object { - "releaseTimestamp": "2018-01-24T17:04:52+0000", - "version": "4.5", - }, Object { "releaseTimestamp": "2018-01-24T17:04:52+0000", "version": "4.5", @@ -448,14 +228,6 @@ Object { "releaseTimestamp": "2018-02-05T13:22:49+0000", "version": "4.5.1", }, - Object { - "releaseTimestamp": "2018-02-05T13:22:49+0000", - "version": "4.5.1", - }, - Object { - "releaseTimestamp": "2018-02-28T13:36:36+0000", - "version": "4.6", - }, Object { "releaseTimestamp": "2018-02-28T13:36:36+0000", "version": "4.6", @@ -464,14 +236,6 @@ Object { "releaseTimestamp": "2018-04-18T09:09:12+0000", "version": "4.7", }, - Object { - "releaseTimestamp": "2018-04-18T09:09:12+0000", - "version": "4.7", - }, - Object { - "releaseTimestamp": "2018-06-04T10:39:58+0000", - "version": "4.8", - }, Object { "releaseTimestamp": "2018-06-04T10:39:58+0000", "version": "4.8", @@ -480,22 +244,10 @@ Object { "releaseTimestamp": "2018-06-21T07:53:06+0000", "version": "4.8.1", }, - Object { - "releaseTimestamp": "2018-06-21T07:53:06+0000", - "version": "4.8.1", - }, Object { "releaseTimestamp": "2018-07-16T08:14:03+0000", "version": "4.9", }, - Object { - "releaseTimestamp": "2018-07-16T08:14:03+0000", - "version": "4.9", - }, - Object { - "releaseTimestamp": "2018-08-27T18:35:06+0000", - "version": "4.10", - }, Object { "releaseTimestamp": "2018-08-27T18:35:06+0000", "version": "4.10", @@ -504,14 +256,6 @@ Object { "releaseTimestamp": "2018-09-12T11:33:27+0000", "version": "4.10.1", }, - Object { - "releaseTimestamp": "2018-09-12T11:33:27+0000", - "version": "4.10.1", - }, - Object { - "releaseTimestamp": "2018-09-19T18:10:15+0000", - "version": "4.10.2", - }, Object { "releaseTimestamp": "2018-09-19T18:10:15+0000", "version": "4.10.2", @@ -520,14 +264,6 @@ Object { "releaseTimestamp": null, "version": "4.10.3", }, - Object { - "releaseTimestamp": null, - "version": "4.10.3", - }, - Object { - "releaseTimestamp": null, - "version": "5.0", - }, Object { "releaseTimestamp": null, "version": "5.0", diff --git a/lib/datasource/gradle-version/index.ts b/lib/datasource/gradle-version/index.ts index b8fb6c3e8ed4e0..aab1d9d61399d5 100644 --- a/lib/datasource/gradle-version/index.ts +++ b/lib/datasource/gradle-version/index.ts @@ -1,20 +1,14 @@ -import is from '@sindresorhus/is'; import { logger } from '../../logger'; import { Http } from '../../util/http'; import { regEx } from '../../util/regex'; -import { - DatasourceError, - GetReleasesConfig, - Release, - ReleaseResult, -} from '../common'; +import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common'; export const id = 'gradle-version'; +export const defaultRegistryUrls = ['https://services.gradle.org/versions/all']; +export const registryStrategy = 'merge'; const http = new Http(id); -const GradleVersionsServiceUrl = 'https://services.gradle.org/versions/all'; - interface GradleRelease { snapshot?: boolean; nightly?: boolean; @@ -38,41 +32,32 @@ function formatBuildTime(timeStr: string): string | null { } export async function getReleases({ - registryUrls, + registryUrl, }: GetReleasesConfig): Promise { - const versionsUrls = is.nonEmptyArray(registryUrls) - ? registryUrls - : [GradleVersionsServiceUrl]; - - const allReleases: Release[][] = await Promise.all( - versionsUrls.map(async (url) => { - try { - const response = await http.getJson(url); - const releases = response.body - .filter((release) => !release.snapshot && !release.nightly) - .filter( - (release) => - // some milestone have wrong metadata and need to be filtered by version name content - release.rcFor === '' && !release.version.includes('milestone') - ) - .map((release) => ({ - version: release.version, - releaseTimestamp: formatBuildTime(release.buildTime), - })); - return releases; - } catch (err) /* istanbul ignore next */ { - // istanbul ignore if - if (err.host === 'services.gradle.org') { - throw new DatasourceError(err); - } - logger.debug({ err }, 'gradle-version err'); - return null; - } - }) - ); + let releases; + try { + const response = await http.getJson(registryUrl); + releases = response.body + .filter((release) => !release.snapshot && !release.nightly) + .filter( + (release) => + // some milestone have wrong metadata and need to be filtered by version name content + release.rcFor === '' && !release.version.includes('milestone') + ) + .map((release) => ({ + version: release.version, + releaseTimestamp: formatBuildTime(release.buildTime), + })); + } catch (err) /* istanbul ignore next */ { + if (err.host === 'services.gradle.org') { + throw new DatasourceError(err); + } + logger.debug({ err }, 'gradle-version err'); + return null; + } const res: ReleaseResult = { - releases: Array.prototype.concat.apply([], allReleases).filter(Boolean), + releases, homepage: 'https://gradle.org', sourceUrl: 'https://github.com/gradle/gradle', }; diff --git a/lib/datasource/helm/index.ts b/lib/datasource/helm/index.ts index 33671e577fff71..4b684275219c16 100644 --- a/lib/datasource/helm/index.ts +++ b/lib/datasource/helm/index.ts @@ -13,6 +13,7 @@ const http = new Http(id); export const defaultRegistryUrls = [ 'https://kubernetes-charts.storage.googleapis.com/', ]; +export const registryStrategy = 'first'; export async function getRepositoryData( repository: string @@ -101,9 +102,8 @@ export async function getRepositoryData( export async function getReleases({ lookupName, - registryUrls, + registryUrl: helmRepository, }: GetReleasesConfig): Promise { - const [helmRepository] = registryUrls; const repositoryData = await getRepositoryData(helmRepository); if (!repositoryData) { logger.debug(`Couldn't get index.yaml file from ${helmRepository}`); diff --git a/lib/datasource/index.spec.ts b/lib/datasource/index.spec.ts index 73b3af5ea8750a..974003a4b0e4e9 100644 --- a/lib/datasource/index.spec.ts +++ b/lib/datasource/index.spec.ts @@ -1,16 +1,28 @@ import { mocked } from '../../test/util'; +import { DATASOURCE_FAILURE } from '../constants/error-messages'; import { loadModules } from '../util/modules'; +import { DatasourceError } from './common'; import * as datasourceDocker from './docker'; import * as datasourceGithubTags from './github-tags'; +import * as datasourceMaven from './maven'; import * as datasourceNpm from './npm'; +import * as datasourcePackagist from './packagist'; import * as datasource from '.'; jest.mock('./docker'); +jest.mock('./maven'); jest.mock('./npm'); +jest.mock('./packagist'); +const dockerDatasource = mocked(datasourceDocker); +const mavenDatasource = mocked(datasourceMaven); const npmDatasource = mocked(datasourceNpm); +const packagistDatasource = mocked(datasourcePackagist); describe('datasource/index', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); it('returns datasources', () => { expect(datasource.getDatasources()).toBeDefined(); expect(datasource.getDatasourceList()).toBeDefined(); @@ -98,6 +110,96 @@ describe('datasource/index', () => { expect(res).toMatchSnapshot(); expect(res.sourceUrl).toBeDefined(); }); + it('warns if multiple registryUrls for registryStrategy=first', async () => { + dockerDatasource.getReleases.mockResolvedValue(null); + const res = await datasource.getPkgReleases({ + datasource: datasourceDocker.id, + depName: 'something', + registryUrls: ['https://docker.com', 'https://docker.io'], + }); + expect(res).toMatchSnapshot(); + }); + it('hunts registries and returns success', async () => { + packagistDatasource.getReleases.mockResolvedValueOnce(null); + packagistDatasource.getReleases.mockResolvedValueOnce({ + releases: [{ version: '1.0.0' }], + }); + const res = await datasource.getPkgReleases({ + datasource: datasourcePackagist.id, + depName: 'something', + registryUrls: ['https://reg1.com', 'https://reg2.io'], + }); + expect(res).not.toBeNull(); + }); + it('hunts registries and aborts on DatasourceError', async () => { + packagistDatasource.getReleases.mockImplementationOnce(() => { + throw new DatasourceError(new Error()); + }); + await expect( + datasource.getPkgReleases({ + datasource: datasourcePackagist.id, + depName: 'something', + registryUrls: ['https://reg1.com', 'https://reg2.io'], + }) + ).rejects.toThrow(DATASOURCE_FAILURE); + }); + it('hunts registries and passes on error', async () => { + packagistDatasource.getReleases.mockImplementationOnce(() => { + throw new Error('a'); + }); + packagistDatasource.getReleases.mockImplementationOnce(() => { + throw new Error('b'); + }); + await expect( + datasource.getPkgReleases({ + datasource: datasourcePackagist.id, + depName: 'something', + registryUrls: ['https://reg1.com', 'https://reg2.io'], + }) + ).rejects.toThrow('b'); + }); + it('merges registries and returns success', async () => { + mavenDatasource.getReleases.mockResolvedValueOnce({ + releases: [{ version: '1.0.0' }, { version: '1.1.0' }], + }); + mavenDatasource.getReleases.mockResolvedValueOnce({ + releases: [{ version: '1.0.0' }], + }); + const res = await datasource.getPkgReleases({ + datasource: datasourceMaven.id, + depName: 'something', + registryUrls: ['https://reg1.com', 'https://reg2.io'], + }); + expect(res).toMatchSnapshot(); + expect(res.releases).toHaveLength(2); + }); + it('merges registries and aborts on DatasourceError', async () => { + mavenDatasource.getReleases.mockImplementationOnce(() => { + throw new DatasourceError(new Error()); + }); + await expect( + datasource.getPkgReleases({ + datasource: datasourceMaven.id, + depName: 'something', + registryUrls: ['https://reg1.com', 'https://reg2.io'], + }) + ).rejects.toThrow(DATASOURCE_FAILURE); + }); + it('merges registries and passes on error', async () => { + mavenDatasource.getReleases.mockImplementationOnce(() => { + throw new Error('a'); + }); + mavenDatasource.getReleases.mockImplementationOnce(() => { + throw new Error('b'); + }); + await expect( + datasource.getPkgReleases({ + datasource: datasourceMaven.id, + depName: 'something', + registryUrls: ['https://reg1.com', 'https://reg2.io'], + }) + ).rejects.toThrow('b'); + }); it('trims sourceUrl', async () => { npmDatasource.getReleases.mockResolvedValue({ sourceUrl: ' https://abc.com', diff --git a/lib/datasource/index.ts b/lib/datasource/index.ts index 8c05a912cdc8ba..9ab408ccfcdf08 100644 --- a/lib/datasource/index.ts +++ b/lib/datasource/index.ts @@ -29,6 +29,102 @@ function load(datasource: string): Promise { type GetReleasesInternalConfig = GetReleasesConfig & GetPkgReleasesConfig; +function firstRegistry( + config: GetReleasesInternalConfig, + datasource: Datasource, + registryUrls: string[] +): Promise { + if (registryUrls.length > 1) { + logger.warn( + { datasource: datasource.id, depName: config.depName, registryUrls }, + 'Excess registryUrls found for datasource lookup - using first configured only' + ); + } + const registryUrl = registryUrls[0]; + return datasource.getReleases({ + ...config, + registryUrl, + }); +} + +async function huntRegistries( + config: GetReleasesInternalConfig, + datasource: Datasource, + registryUrls: string[] +): Promise { + let res: ReleaseResult; + let datasourceError; + for (const registryUrl of registryUrls) { + try { + res = await datasource.getReleases({ + ...config, + registryUrl, + }); + if (res) { + break; + } + } catch (err) { + if (err instanceof DatasourceError) { + throw err; + } + // We'll always save the last-thrown error + datasourceError = err; + logger.trace({ err }, 'datasource hunt failure'); + } + } + if (res === undefined && datasourceError) { + // if we failed to get a result and also got an error then throw it + throw datasourceError; + } + return res; +} + +async function mergeRegistries( + config: GetReleasesInternalConfig, + datasource: Datasource, + registryUrls: string[] +): Promise { + let combinedRes: ReleaseResult; + let datasourceError; + for (const registryUrl of registryUrls) { + try { + const res = await datasource.getReleases({ + ...config, + registryUrl, + }); + if (combinedRes) { + combinedRes = { ...res, ...combinedRes }; + combinedRes.releases = [...combinedRes.releases, ...res.releases]; + } else { + combinedRes = res; + } + } catch (err) { + if (err instanceof DatasourceError) { + throw err; + } + // We'll always save the last-thrown error + datasourceError = err; + logger.trace({ err }, 'datasource merge failure'); + } + } + if (combinedRes === undefined && datasourceError) { + // if we failed to get a result and also got an error then throw it + throw datasourceError; + } + // De-duplicate releases + if (combinedRes?.releases?.length) { + const seenVersions = new Set(); + combinedRes.releases = combinedRes.releases.filter((release) => { + if (seenVersions.has(release.version)) { + return false; + } + seenVersions.add(release.version); + return true; + }); + } + return combinedRes; +} + function resolveRegistryUrls( datasource: Datasource, extractedUrls: string[] @@ -49,12 +145,31 @@ async function fetchReleases( } const datasource = await load(datasourceName); const registryUrls = resolveRegistryUrls(datasource, config.registryUrls); - let dep = await datasource.getReleases({ - ...config, - registryUrls, - }); - if (!(dep && dep.releases.length)) { - dep = null; + let dep: ReleaseResult; + if (datasource.registryStrategy) { + // istanbul ignore if + if (!registryUrls.length) { + logger.warn( + { datasource: datasourceName, depName: config.depName }, + 'Missing registryUrls for registryStrategy' + ); + return null; + } + if (datasource.registryStrategy === 'first') { + dep = await firstRegistry(config, datasource, registryUrls); + } else if (datasource.registryStrategy === 'hunt') { + dep = await huntRegistries(config, datasource, registryUrls); + } else if (datasource.registryStrategy === 'merge') { + dep = await mergeRegistries(config, datasource, registryUrls); + } + } else { + dep = await datasource.getReleases({ + ...config, + registryUrls, + }); + } + if (!dep?.releases?.length) { + return null; } addMetaData(dep, datasourceName, config.lookupName); return dep; @@ -131,10 +246,11 @@ export async function getDigest( config: DigestConfig, value?: string ): Promise { + const datasource = await load(config.datasource); const lookupName = config.lookupName || config.depName; - const { registryUrls } = config; - return (await load(config.datasource)).getDigest( - { lookupName, registryUrls }, + const registryUrls = resolveRegistryUrls(datasource, config.registryUrls); + return datasource.getDigest( + { lookupName, registryUrl: registryUrls[0] }, value ); } diff --git a/lib/datasource/maven/index.ts b/lib/datasource/maven/index.ts index 99efb384dde0ef..2569c7240ef043 100644 --- a/lib/datasource/maven/index.ts +++ b/lib/datasource/maven/index.ts @@ -13,6 +13,7 @@ import { downloadHttpProtocol, isHttpResourceExists } from './util'; export { id } from './common'; export const defaultRegistryUrls = [MAVEN_REPO]; +export const registryStrategy = 'merge'; function containsPlaceholder(str: string): boolean { return /\${.*?}/g.test(str); @@ -250,53 +251,39 @@ async function filterMissingArtifacts( export async function getReleases({ lookupName, - registryUrls, + registryUrl, }: GetReleasesConfig): Promise { - const repositories = registryUrls.map((repository) => - repository.replace(/\/?$/, '/') - ); const dependency = getDependencyParts(lookupName); const versions: string[] = []; const repoForVersions = {}; - for (let i = 0; i < repositories.length; i += 1) { - const repoUrl = repositories[i]; - logger.debug( - `Looking up ${dependency.display} in repository #${i} - ${repoUrl}` + const repoUrl = registryUrl.replace(/\/?$/, '/'); + logger.debug(`Looking up ${dependency.display} in repository ${repoUrl}`); + const metadataVersions = await getVersionsFromMetadata(dependency, repoUrl); + if (metadataVersions) { + const availableVersions = await filterMissingArtifacts( + dependency, + repoUrl, + metadataVersions ); - const metadataVersions = await getVersionsFromMetadata(dependency, repoUrl); - if (metadataVersions) { - const availableVersions = await filterMissingArtifacts( - dependency, - repoUrl, - metadataVersions - ); - const filteredVersions = availableVersions.filter( - (version) => !versions.includes(version) - ); - versions.push(...filteredVersions); - - const latestVersion = getLatestStableVersion(filteredVersions); - if (latestVersion) { - repoForVersions[latestVersion] = repoUrl; - } + const filteredVersions = availableVersions.filter( + (version) => !versions.includes(version) + ); + versions.push(...filteredVersions); - logger.debug(`Found ${availableVersions.length} new versions for ${dependency.display} in repository ${repoUrl}`); // prettier-ignore + const latestVersion = getLatestStableVersion(filteredVersions); + if (latestVersion) { + repoForVersions[latestVersion] = repoUrl; } - } - if (versions.length === 0) { - logger.debug(`No versions found for ${dependency.display} in ${repositories.length} repositories`); // prettier-ignore - return null; + logger.debug(`Found ${availableVersions.length} new versions for ${dependency.display} in repository ${repoUrl}`); // prettier-ignore } - logger.debug(`Found ${versions.length} versions for ${dependency.display}`); let dependencyInfo = {}; const latestVersion = getLatestStableVersion(versions); if (latestVersion) { - const repoUrl = repoForVersions[latestVersion]; dependencyInfo = await getDependencyInfo( dependency, - repoUrl, + repoForVersions[latestVersion], latestVersion ); } diff --git a/lib/datasource/nuget/index.ts b/lib/datasource/nuget/index.ts index 3fcb9b5f4fdfc6..48326b8d5ff01a 100644 --- a/lib/datasource/nuget/index.ts +++ b/lib/datasource/nuget/index.ts @@ -1,6 +1,5 @@ import urlApi from 'url'; import { logger } from '../../logger'; -import { clone } from '../../util/clone'; import { GetReleasesConfig, ReleaseResult } from '../common'; import * as v2 from './v2'; import * as v3 from './v3'; @@ -8,6 +7,7 @@ import * as v3 from './v3'; export { id } from './common'; export const defaultRegistryUrls = [v3.getDefaultFeed()]; +export const registryStrategy = 'merge'; function parseRegistryUrl( registryUrl: string @@ -32,43 +32,18 @@ function parseRegistryUrl( export async function getReleases({ lookupName, - registryUrls, + registryUrl, }: GetReleasesConfig): Promise { logger.trace(`nuget.getReleases(${lookupName})`); - let dep: ReleaseResult = null; - for (const feed of registryUrls) { - const { feedUrl, protocolVersion } = parseRegistryUrl(feed); - let res: ReleaseResult = null; - if (protocolVersion === 2) { - res = await v2.getReleases(feedUrl, lookupName); - } else if (protocolVersion === 3) { - const queryUrl = await v3.getQueryUrl(feedUrl); - if (queryUrl !== null) { - res = await v3.getReleases(feedUrl, queryUrl, lookupName); - } - } - if (res !== null) { - res = clone(res); - if (dep !== null) { - for (const resRelease of res.releases) { - if ( - !dep.releases.find( - (depRelease) => depRelease.version === resRelease.version - ) - ) { - dep.releases.push(resRelease); - } - } - } else { - dep = res; - } - } + const { feedUrl, protocolVersion } = parseRegistryUrl(registryUrl); + if (protocolVersion === 2) { + return v2.getReleases(feedUrl, lookupName); } - if (dep === null) { - logger.debug( - { lookupName }, - `Dependency lookup failure: not found in all feeds` - ); + if (protocolVersion === 3) { + const queryUrl = await v3.getQueryUrl(feedUrl); + if (queryUrl !== null) { + return v3.getReleases(feedUrl, queryUrl, lookupName); + } } - return dep; + return null; } diff --git a/lib/datasource/packagist/index.ts b/lib/datasource/packagist/index.ts index 73d8d3e238ff7b..77946c2bb4cb3d 100644 --- a/lib/datasource/packagist/index.ts +++ b/lib/datasource/packagist/index.ts @@ -1,5 +1,4 @@ import URL from 'url'; -import is from '@sindresorhus/is'; import pAll from 'p-all'; import { logger } from '../../logger'; @@ -10,6 +9,8 @@ import { Http, HttpOptions } from '../../util/http'; import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common'; export const id = 'packagist'; +export const defaultRegistryUrls = ['https://packagist.org']; +export const registryStrategy = 'hunt'; const http = new Http(id); @@ -325,19 +326,8 @@ async function packageLookup( export async function getReleases({ lookupName, - registryUrls, + registryUrl, }: GetReleasesConfig): Promise { logger.trace(`getReleases(${lookupName})`); - - let res: ReleaseResult; - const registries = is.nonEmptyArray(registryUrls) - ? registryUrls - : ['https://packagist.org']; - for (const regUrl of registries) { - res = await packageLookup(regUrl, lookupName); - if (res) { - break; - } - } - return res; + return packageLookup(registryUrl, lookupName); } diff --git a/lib/datasource/pod/index.ts b/lib/datasource/pod/index.ts index ce957dad14780c..24b0361f875175 100644 --- a/lib/datasource/pod/index.ts +++ b/lib/datasource/pod/index.ts @@ -8,6 +8,7 @@ import { GetReleasesConfig, ReleaseResult } from '../common'; export const id = 'pod'; export const defaultRegistryUrls = ['https://cdn.cocoapods.org']; +export const registryStrategy = 'hunt'; const cacheNamespace = `datasource-${id}`; const cacheMinutes = 30; @@ -149,39 +150,36 @@ function isDefaultRepo(url: string): boolean { export async function getReleases({ lookupName, - registryUrls, + registryUrl, }: GetReleasesConfig): Promise { const podName = lookupName.replace(/\/.*$/, ''); const cachedResult = await globalCache.get( cacheNamespace, - podName + registryUrl + podName ); - /* istanbul ignore next line */ - if (cachedResult) { - logger.debug(`CocoaPods: Return cached result for ${podName}`); + + // istanbul ignore if + if (cachedResult !== undefined) { + logger.trace(`CocoaPods: Return cached result for ${podName}`); return cachedResult; } - let result: ReleaseResult | null = null; - for (let idx = 0; !result && idx < registryUrls.length; idx += 1) { - let registryUrl = registryUrls[idx].replace(/\/+$/, ''); - - // In order to not abuse github API limits, query CDN instead - if (isDefaultRepo(registryUrl)) { - [registryUrl] = defaultRegistryUrls; - } + let baseUrl = registryUrl.replace(/\/+$/, ''); - if (githubRegex.exec(registryUrl)) { - result = await getReleasesFromGithub(podName, registryUrl); - } else { - result = await getReleasesFromCDN(podName, registryUrl); - } + // In order to not abuse github API limits, query CDN instead + if (isDefaultRepo(baseUrl)) { + [baseUrl] = defaultRegistryUrls; } - if (result) { - await globalCache.set(cacheNamespace, podName, result, cacheMinutes); + let result: ReleaseResult | null = null; + if (githubRegex.exec(baseUrl)) { + result = await getReleasesFromGithub(podName, baseUrl); + } else { + result = await getReleasesFromCDN(podName, baseUrl); } + await globalCache.set(cacheNamespace, podName, result, cacheMinutes); + return result; } diff --git a/lib/datasource/pypi/__snapshots__/index.spec.ts.snap b/lib/datasource/pypi/__snapshots__/index.spec.ts.snap index 70c17a124ead33..957cebdae4987f 100644 --- a/lib/datasource/pypi/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/pypi/__snapshots__/index.spec.ts.snap @@ -394,21 +394,6 @@ Array [ ] `; -exports[`datasource/pypi getReleases supports custom datasource url from environmental variable 1`] = ` -Array [ - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate", - "host": "my.pypi.python", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://my.pypi.python/pypi/azure-cli-monitor/json", - }, -] -`; - exports[`datasource/pypi getReleases supports multiple custom datasource urls 1`] = ` Array [ Object { diff --git a/lib/datasource/pypi/index.spec.ts b/lib/datasource/pypi/index.spec.ts index 27b28e9e33864b..e3c577caf38275 100644 --- a/lib/datasource/pypi/index.spec.ts +++ b/lib/datasource/pypi/index.spec.ts @@ -82,20 +82,6 @@ describe('datasource/pypi', () => { }); expect(httpMock.getTrace()).toMatchSnapshot(); }); - it('supports custom datasource url from environmental variable', async () => { - httpMock - .scope('https://my.pypi.python/pypi/') - .get('/azure-cli-monitor/json') - .reply(200, JSON.parse(res1)); - const pipIndexUrl = process.env.PIP_INDEX_URL; - process.env.PIP_INDEX_URL = 'https://my.pypi.python/pypi/'; - await getPkgReleases({ - datasource, - depName: 'azure-cli-monitor', - }); - expect(httpMock.getTrace()).toMatchSnapshot(); - process.env.PIP_INDEX_URL = pipIndexUrl; - }); it('supports multiple custom datasource urls', async () => { httpMock .scope('https://custom.pypi.net/foo') diff --git a/lib/datasource/pypi/index.ts b/lib/datasource/pypi/index.ts index 311a98d32e6394..a351082324e22c 100644 --- a/lib/datasource/pypi/index.ts +++ b/lib/datasource/pypi/index.ts @@ -1,14 +1,19 @@ import url from 'url'; -import is from '@sindresorhus/is'; import changelogFilenameRegex from 'changelog-filename-regex'; import { parse } from 'node-html-parser'; import { logger } from '../../logger'; import { Http } from '../../util/http'; +import { ensureTrailingSlash } from '../../util/url'; import { matches } from '../../versioning/pep440'; import * as pep440 from '../../versioning/pep440'; import { GetReleasesConfig, ReleaseResult } from '../common'; export const id = 'pypi'; +export const defaultRegistryUrls = [ + process.env.PIP_INDEX_URL || 'https://pypi.org/pypi/', +]; +export const registryStrategy = 'hunt'; + const github_repo_pattern = /^https?:\/\/github\.com\/[^\\/]+\/[^\\/]+$/; const http = new Http(id); @@ -208,36 +213,13 @@ async function getSimpleDependency( export async function getReleases({ compatibility, lookupName, - registryUrls, + registryUrl, }: GetReleasesConfig): Promise { - let hostUrls = ['https://pypi.org/pypi/']; - if (is.nonEmptyArray(registryUrls)) { - hostUrls = registryUrls; - } - if (process.env.PIP_INDEX_URL) { - hostUrls = [process.env.PIP_INDEX_URL]; - } - let dep: ReleaseResult; - for (let index = 0; index < hostUrls.length && !dep; index += 1) { - let hostUrl = hostUrls[index]; - hostUrl += hostUrl.endsWith('/') ? '' : '/'; - if (hostUrl.endsWith('/simple/') || hostUrl.endsWith('/+simple/')) { - logger.trace( - { lookupName, hostUrl }, - 'Looking up pypi simple dependency' - ); - dep = await getSimpleDependency(lookupName, hostUrl); - } else { - logger.trace({ lookupName, hostUrl }, 'Looking up pypi api dependency'); - dep = await getDependency(lookupName, hostUrl, compatibility); - } - if (dep !== null) { - logger.trace({ lookupName, hostUrl }, 'Found pypi result'); - } + const hostUrl = ensureTrailingSlash(registryUrl); + if (hostUrl.endsWith('/simple/') || hostUrl.endsWith('/+simple/')) { + logger.trace({ lookupName, hostUrl }, 'Looking up pypi simple dependency'); + return getSimpleDependency(lookupName, hostUrl); } - if (dep) { - return dep; - } - logger.debug({ lookupName, registryUrls }, 'No pypi result - returning null'); - return null; + logger.trace({ lookupName, hostUrl }, 'Looking up pypi api dependency'); + return getDependency(lookupName, hostUrl, compatibility); } diff --git a/lib/datasource/rubygems/index.ts b/lib/datasource/rubygems/index.ts index ddc9e60c98c4c0..336ffdae7c8a1e 100644 --- a/lib/datasource/rubygems/index.ts +++ b/lib/datasource/rubygems/index.ts @@ -1,3 +1,4 @@ export { getReleases } from './releases'; export { id } from './common'; export const defaultRegistryUrls = ['https://rubygems.org']; +export const registryStrategy = 'hunt'; diff --git a/lib/datasource/rubygems/releases.ts b/lib/datasource/rubygems/releases.ts index 0be99f712a626e..c7458eb53e02f4 100644 --- a/lib/datasource/rubygems/releases.ts +++ b/lib/datasource/rubygems/releases.ts @@ -4,20 +4,11 @@ import { getRubygemsOrgDependency } from './get-rubygems-org'; export async function getReleases({ lookupName, - registryUrls, + registryUrl, }: GetReleasesConfig): Promise { - for (const registry of registryUrls) { - let pkg: ReleaseResult; - // prettier-ignore - if (registry.endsWith('rubygems.org')) { // lgtm [js/incomplete-url-substring-sanitization] - pkg = await getRubygemsOrgDependency(lookupName); - } else { - pkg = await getDependency({ dependency: lookupName, registry }); + // prettier-ignore + if (registryUrl.endsWith('rubygems.org')) { // lgtm [js/incomplete-url-substring-sanitization] + return getRubygemsOrgDependency(lookupName); } - if (pkg) { - return pkg; - } - } - - return null; + return getDependency({ dependency: lookupName, registry: registryUrl }); } diff --git a/lib/datasource/sbt-package/index.ts b/lib/datasource/sbt-package/index.ts index 4067bfc229f173..e5ea80a6d9359d 100644 --- a/lib/datasource/sbt-package/index.ts +++ b/lib/datasource/sbt-package/index.ts @@ -8,6 +8,7 @@ import { parseIndexDir } from '../sbt-plugin/util'; export const id = 'sbt-package'; export const defaultRegistryUrls = [MAVEN_REPO]; +export const registryStrategy = 'hunt'; const ensureTrailingSlash = (str: string): string => str.replace(/\/?$/, '/'); @@ -65,20 +66,18 @@ export async function resolvePackageReleases( export async function getReleases({ lookupName, - registryUrls, + registryUrl, }: GetReleasesConfig): Promise { const [groupId, artifactId] = lookupName.split(':'); const groupIdSplit = groupId.split('.'); const artifactIdSplit = artifactId.split('_'); const [artifact, scalaVersion] = artifactIdSplit; - const repoRoots = registryUrls.map((x) => x.replace(/\/?$/, '')); + const repoRoot = ensureTrailingSlash(registryUrl); const searchRoots: string[] = []; - repoRoots.forEach((repoRoot) => { - // Optimize lookup order - searchRoots.push(`${repoRoot}/${groupIdSplit.join('/')}`); - searchRoots.push(`${repoRoot}/${groupIdSplit.join('.')}`); - }); + // Optimize lookup order + searchRoots.push(`${repoRoot}${groupIdSplit.join('/')}`); + searchRoots.push(`${repoRoot}${groupIdSplit.join('.')}`); for (let idx = 0; idx < searchRoots.length; idx += 1) { const searchRoot = searchRoots[idx]; diff --git a/lib/datasource/sbt-plugin/index.ts b/lib/datasource/sbt-plugin/index.ts index 670c0b615bfbd2..b4a5aea9356954 100644 --- a/lib/datasource/sbt-plugin/index.ts +++ b/lib/datasource/sbt-plugin/index.ts @@ -8,6 +8,7 @@ import { SBT_PLUGINS_REPO, parseIndexDir } from './util'; export const id = 'sbt-plugin'; export const defaultRegistryUrls = [SBT_PLUGINS_REPO]; +export const registryStrategy = 'hunt'; const ensureTrailingSlash = (str: string): string => str.replace(/\/?$/, '/'); @@ -62,20 +63,18 @@ async function resolvePluginReleases( export async function getReleases({ lookupName, - registryUrls, + registryUrl, }: GetReleasesConfig): Promise { const [groupId, artifactId] = lookupName.split(':'); const groupIdSplit = groupId.split('.'); const artifactIdSplit = artifactId.split('_'); const [artifact, scalaVersion] = artifactIdSplit; - const repoRoots = registryUrls.map((x) => x.replace(/\/?$/, '')); + const repoRoot = ensureTrailingSlash(registryUrl); const searchRoots: string[] = []; - repoRoots.forEach((repoRoot) => { - // Optimize lookup order - searchRoots.push(`${repoRoot}/${groupIdSplit.join('.')}`); - searchRoots.push(`${repoRoot}/${groupIdSplit.join('/')}`); - }); + // Optimize lookup order + searchRoots.push(`${repoRoot}${groupIdSplit.join('.')}`); + searchRoots.push(`${repoRoot}${groupIdSplit.join('/')}`); for (let idx = 0; idx < searchRoots.length; idx += 1) { const searchRoot = searchRoots[idx]; diff --git a/lib/datasource/terraform-module/index.ts b/lib/datasource/terraform-module/index.ts index 62498efef9bb95..da0bce26adee16 100644 --- a/lib/datasource/terraform-module/index.ts +++ b/lib/datasource/terraform-module/index.ts @@ -1,10 +1,11 @@ -import is from '@sindresorhus/is'; import { logger } from '../../logger'; import * as globalCache from '../../util/cache/global'; import { Http } from '../../util/http'; import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common'; export const id = 'terraform-module'; +export const defaultRegistryUrls = ['https://registry.terraform.io']; +export const registryStrategy = 'first'; const http = new Http(id); @@ -15,17 +16,15 @@ interface RegistryRepository { function getRegistryRepository( lookupName: string, - registryUrls: string[] + registryUrl: string ): RegistryRepository { let registry: string; const split = lookupName.split('/'); if (split.length > 3 && split[0].includes('.')) { [registry] = split; split.shift(); - } else if (is.nonEmptyArray(registryUrls)) { - [registry] = registryUrls; } else { - registry = 'registry.terraform.io'; + registry = registryUrl; } if (!/^https?:\/\//.test(registry)) { registry = `https://${registry}`; @@ -54,11 +53,11 @@ interface TerraformRelease { */ export async function getReleases({ lookupName, - registryUrls, + registryUrl, }: GetReleasesConfig): Promise { const { registry, repository } = getRegistryRepository( lookupName, - registryUrls + registryUrl ); logger.debug( { registry, terraformRepository: repository }, diff --git a/lib/datasource/terraform-provider/index.ts b/lib/datasource/terraform-provider/index.ts index abe32ad04aa3a4..f818251dad686e 100644 --- a/lib/datasource/terraform-provider/index.ts +++ b/lib/datasource/terraform-provider/index.ts @@ -22,7 +22,6 @@ interface TerraformProvider { */ export async function getReleases({ lookupName, - registryUrls, }: GetReleasesConfig): Promise { const repository = `hashicorp/${lookupName}`;