From 3ba1c939604ec81d6c7d1488456a1b052da50630 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 22 Jun 2020 13:30:00 +0200 Subject: [PATCH 01/25] chore(deps): update dependency eslint-plugin-import to v2.21.2 (#6505) Co-authored-by: Renovate Bot Co-authored-by: Rhys Arkins --- package.json | 2 +- tools/check-re2.mjs | 2 +- yarn.lock | 61 +++++++++++++++++++++++++++++++-------------- 3 files changed, 44 insertions(+), 21 deletions(-) diff --git a/package.json b/package.json index 9375a355ae5d9f..26200f6d77e103 100644 --- a/package.json +++ b/package.json @@ -230,7 +230,7 @@ "eslint": "6.8.0", "eslint-config-airbnb-typescript": "6.3.2", "eslint-config-prettier": "6.11.0", - "eslint-plugin-import": "2.20.2", + "eslint-plugin-import": "2.21.2", "eslint-plugin-jest": "23.13.2", "eslint-plugin-promise": "4.2.1", "glob": "7.1.6", diff --git a/tools/check-re2.mjs b/tools/check-re2.mjs index 0d55da98a28bd4..04a90c4c3a2a95 100644 --- a/tools/check-re2.mjs +++ b/tools/check-re2.mjs @@ -4,7 +4,7 @@ import shell from 'shelljs'; (async () => { shell.echo('-n', 'Checking re2 ... '); try { - const { default: RE2 } = await import('re2'); + const { default: RE2 } = await import('re2'); // eslint-disable-line import/no-extraneous-dependencies new RE2('.*').exec('test'); shell.echo(`ok.`); } catch (e) { diff --git a/yarn.lock b/yarn.lock index 8dffe5d2877b44..601f72e2f62982 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1583,6 +1583,11 @@ resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.30.tgz#44cb52f32a809734ca562e685c6473b5754a7818" integrity sha512-sqm9g7mHlPY/43fcSNrCYfOeX9zkTTK+euO5E6+CVijSMm5tTjkVdwdqRkY3ljjIAf8679vps5jKUoJBCLsMDA== +"@types/json5@^0.0.29": + version "0.0.29" + resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" + integrity sha1-7ihweulOEdK4J7y+UnC86n8+ce4= + "@types/later@1.2.6": version "1.2.6" resolved "https://registry.yarnpkg.com/@types/later/-/later-1.2.6.tgz#8ea70d292e5fb880039a7ed13da19a24c9e35a56" @@ -2046,7 +2051,7 @@ array-ify@^1.0.0: resolved "https://registry.yarnpkg.com/array-ify/-/array-ify-1.0.0.tgz#9e528762b4a9066ad163a6962a364418e9626ece" integrity sha1-nlKHYrSpBmrRY6aWKjZEGOlibs4= -array-includes@^3.0.3: +array-includes@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/array-includes/-/array-includes-3.1.1.tgz#cdd67e6852bdf9c1215460786732255ed2459348" integrity sha512-c2VXaCHl7zPsvpkFsw4nxvFie4fh1ur9bpcgsVkIjqn0H/Xwdg+7fv3n2r/isyS8EBj5b06M9kHyZuIr4El6WQ== @@ -2065,7 +2070,7 @@ array-unique@^0.3.2: resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428" integrity sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg= -array.prototype.flat@^1.2.1: +array.prototype.flat@^1.2.3: version "1.2.3" resolved "https://registry.yarnpkg.com/array.prototype.flat/-/array.prototype.flat-1.2.3.tgz#0de82b426b0318dbfdb940089e38b043d37f6c7b" integrity sha512-gBlRZV0VSmfPIeWfuuy56XZMvbVfbEUnOXUvt3F/eUUUSyzlgLxhEX4YAEpxNAogRGehPSnfXyPtYyKAhkzQhQ== @@ -3673,15 +3678,15 @@ eslint-config-prettier@6.11.0: dependencies: get-stdin "^6.0.0" -eslint-import-resolver-node@^0.3.2: - version "0.3.3" - resolved "https://registry.yarnpkg.com/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.3.tgz#dbaa52b6b2816b50bc6711af75422de808e98404" - integrity sha512-b8crLDo0M5RSe5YG8Pu2DYBj71tSB6OvXkfzwbJU2w7y8P4/yo0MyF8jU26IEuEuHF2K5/gcAJE3LhQGqBBbVg== +eslint-import-resolver-node@^0.3.3: + version "0.3.4" + resolved "https://registry.yarnpkg.com/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.4.tgz#85ffa81942c25012d8231096ddf679c03042c717" + integrity sha512-ogtf+5AB/O+nM6DIeBUNr2fuT7ot9Qg/1harBfBtaP13ekEWFQEEMP94BCB7zaNW3gyY+8SHYF00rnqYwXKWOA== dependencies: debug "^2.6.9" resolve "^1.13.1" -eslint-module-utils@^2.4.1: +eslint-module-utils@^2.6.0: version "2.6.0" resolved "https://registry.yarnpkg.com/eslint-module-utils/-/eslint-module-utils-2.6.0.tgz#579ebd094f56af7797d19c9866c9c9486629bfa6" integrity sha512-6j9xxegbqe8/kZY8cYpcp0xhbK0EgJlg3g9mib3/miLaExuuwc3n5UEfSnU6hWMbT0FAYVvDbL9RrRgpUeQIvA== @@ -3689,23 +3694,24 @@ eslint-module-utils@^2.4.1: debug "^2.6.9" pkg-dir "^2.0.0" -eslint-plugin-import@2.20.2: - version "2.20.2" - resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.20.2.tgz#91fc3807ce08be4837141272c8b99073906e588d" - integrity sha512-FObidqpXrR8OnCh4iNsxy+WACztJLXAHBO5hK79T1Hc77PgQZkyDGA5Ag9xAvRpglvLNxhH/zSmZ70/pZ31dHg== +eslint-plugin-import@2.21.2: + version "2.21.2" + resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.21.2.tgz#8fef77475cc5510801bedc95f84b932f7f334a7c" + integrity sha512-FEmxeGI6yaz+SnEB6YgNHlQK1Bs2DKLM+YF+vuTk5H8J9CLbJLtlPvRFgZZ2+sXiKAlN5dpdlrWOjK8ZoZJpQA== dependencies: - array-includes "^3.0.3" - array.prototype.flat "^1.2.1" + array-includes "^3.1.1" + array.prototype.flat "^1.2.3" contains-path "^0.1.0" debug "^2.6.9" doctrine "1.5.0" - eslint-import-resolver-node "^0.3.2" - eslint-module-utils "^2.4.1" + eslint-import-resolver-node "^0.3.3" + eslint-module-utils "^2.6.0" has "^1.0.3" minimatch "^3.0.4" - object.values "^1.1.0" + object.values "^1.1.1" read-pkg-up "^2.0.0" - resolve "^1.12.0" + resolve "^1.17.0" + tsconfig-paths "^3.9.0" eslint-plugin-jest@23.13.2: version "23.13.2" @@ -5959,6 +5965,13 @@ json5@2.1.3, json5@^2.1.2: dependencies: minimist "^1.2.5" +json5@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.1.tgz#779fb0018604fa854eacbf6252180d83543e3dbe" + integrity sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow== + dependencies: + minimist "^1.2.0" + jsonfile@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.0.1.tgz#98966cba214378c8c84b82e085907b40bf614179" @@ -7486,7 +7499,7 @@ object.pick@^1.3.0: dependencies: isobject "^3.0.1" -object.values@^1.1.0: +object.values@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/object.values/-/object.values-1.1.1.tgz#68a99ecde356b7e9295a3c5e0ce31dc8c953de5e" integrity sha512-WTa54g2K8iu0kmS/us18jEmdv1a4Wi//BZ/DTVYEcH0XhLM5NYdpDHja3gt57VrZLcNAO2WGA+KpWsDBaHt6eA== @@ -8691,7 +8704,7 @@ resolve-url@^0.2.1: resolved "https://registry.yarnpkg.com/resolve-url/-/resolve-url-0.2.1.tgz#2c637fe77c893afd2a663fe21aa9080068e2052a" integrity sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo= -resolve@^1.1.6, resolve@^1.10.0, resolve@^1.12.0, resolve@^1.13.1, resolve@^1.17.0, resolve@^1.3.2: +resolve@^1.1.6, resolve@^1.10.0, resolve@^1.13.1, resolve@^1.17.0, resolve@^1.3.2: version "1.17.0" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.17.0.tgz#b25941b54968231cc2d1bb76a79cb7f2c0bf8444" integrity sha512-ic+7JYiV8Vi2yzQGFWOkiZD5Z9z7O2Zhm9XMaTxdJExKasieFCr+yXZ/WmXsckHiKl12ar0y6XiXDx3m4RHn1w== @@ -9861,6 +9874,16 @@ ts-essentials@^4.0.0: resolved "https://registry.yarnpkg.com/ts-essentials/-/ts-essentials-4.0.0.tgz#506c42b270bbd0465574b90416533175b09205ab" integrity sha512-uQJX+SRY9mtbKU+g9kl5Fi7AEMofPCvHfJkQlaygpPmHPZrtgaBqbWFOYyiA47RhnSwwnXdepUJrgqUYxoUyhQ== +tsconfig-paths@^3.9.0: + version "3.9.0" + resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.9.0.tgz#098547a6c4448807e8fcb8eae081064ee9a3c90b" + integrity sha512-dRcuzokWhajtZWkQsDVKbWyY+jgcLC5sqJhg2PSgf4ZkH2aHPvaOY8YWGhmjb68b5qqTfasSsDO9k7RUiEmZAw== + dependencies: + "@types/json5" "^0.0.29" + json5 "^1.0.1" + minimist "^1.2.0" + strip-bom "^3.0.0" + tslib@^1.8.1, tslib@^1.9.0: version "1.13.0" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.13.0.tgz#c881e13cc7015894ed914862d276436fa9a47043" From f77b7e878166a1d08850ba713271e6fea6fdf34c Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Mon, 22 Jun 2020 17:00:46 +0200 Subject: [PATCH 02/25] refactor: simplify global init (#6452) Co-authored-by: Michael Kriese --- lib/workers/global/index.ts | 58 +++++++++++--------------------- lib/workers/global/initialize.ts | 45 +++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 38 deletions(-) create mode 100644 lib/workers/global/initialize.ts diff --git a/lib/workers/global/index.ts b/lib/workers/global/index.ts index c2675d672d60c5..81c618625ba432 100644 --- a/lib/workers/global/index.ts +++ b/lib/workers/global/index.ts @@ -1,41 +1,18 @@ -import os from 'os'; import path from 'path'; import is from '@sindresorhus/is'; import fs from 'fs-extra'; import * as configParser from '../../config'; import { getErrors, logger, setMeta } from '../../logger'; -import { initPlatform } from '../../platform'; import { setUtilConfig } from '../../util'; -import * as globalCache from '../../util/cache/global'; -import { setEmojiConfig } from '../../util/emoji'; import * as hostRules from '../../util/host-rules'; import * as repositoryWorker from '../repository'; import { autodiscoverRepositories } from './autodiscover'; +import { globalFinalize, globalInitialize } from './initialize'; import * as limits from './limits'; type RenovateConfig = configParser.RenovateConfig; type RenovateRepository = configParser.RenovateRepository; -async function setDirectories(input: RenovateConfig): Promise { - const config: RenovateConfig = { ...input }; - process.env.TMPDIR = process.env.RENOVATE_TMPDIR || os.tmpdir(); - if (config.baseDir) { - logger.debug('Using configured baseDir: ' + config.baseDir); - } else { - config.baseDir = path.join(process.env.TMPDIR, 'renovate'); - logger.debug('Using baseDir: ' + config.baseDir); - } - await fs.ensureDir(config.baseDir); - if (config.cacheDir) { - logger.debug('Using configured cacheDir: ' + config.cacheDir); - } else { - config.cacheDir = path.join(config.baseDir, 'cache'); - logger.debug('Using cacheDir: ' + config.cacheDir); - } - await fs.ensureDir(config.cacheDir); - return config; -} - export async function getRepositoryConfig( globalConfig: RenovateConfig, repository: RenovateRepository @@ -57,22 +34,26 @@ function getGlobalConfig(): Promise { return configParser.parseConfigs(process.env, process.argv); } +function haveReachedLimits(): boolean { + if (limits.getLimitRemaining('prCommitsPerRunLimit') <= 0) { + logger.info('Max commits created for this run.'); + return true; + } + return false; +} + export async function start(): Promise<0 | 1> { + let config: RenovateConfig; try { - let config = await getGlobalConfig(); - config = await initPlatform(config); - config = await setDirectories(config); - globalCache.init(config); + // read global config from file, env and cli args + config = await getGlobalConfig(); + // initialize all submodules + config = await globalInitialize(config); + // autodiscover repositories (needs to come after platform initialization) config = await autodiscoverRepositories(config); - - limits.init(config); - setEmojiConfig(config); // Iterate through repositories sequentially for (const repository of config.repositories) { - if (limits.getLimitRemaining('prCommitsPerRunLimit') <= 0) { - logger.debug( - 'Max commits created for this run. Skipping all remaining repositories.' - ); + if (haveReachedLimits()) { break; } const repoConfig = await getRepositoryConfig(config, repository); @@ -83,16 +64,17 @@ export async function start(): Promise<0 | 1> { repoConfig.hostRules = []; } await repositoryWorker.renovateRepository(repoConfig); + setMeta({}); } - setMeta({}); - globalCache.cleanup(config); - logger.debug(`Renovate exiting successfully`); } catch (err) /* istanbul ignore next */ { if (err.message.startsWith('Init: ')) { logger.fatal(err.message.substring(6)); } else { logger.fatal({ err }, `Fatal error: ${err.message}`); } + } finally { + globalFinalize(config); + logger.debug(`Renovate exiting`); } const loggerErrors = getErrors(); /* istanbul ignore if */ diff --git a/lib/workers/global/initialize.ts b/lib/workers/global/initialize.ts new file mode 100644 index 00000000000000..4e36e1f54c45c4 --- /dev/null +++ b/lib/workers/global/initialize.ts @@ -0,0 +1,45 @@ +import os from 'os'; +import path from 'path'; +import fs from 'fs-extra'; +import { RenovateConfig } from '../../config/common'; +import { logger } from '../../logger'; +import { initPlatform } from '../../platform'; +import * as globalCache from '../../util/cache/global'; +import { setEmojiConfig } from '../../util/emoji'; +import * as limits from './limits'; + +async function setDirectories(input: RenovateConfig): Promise { + const config: RenovateConfig = { ...input }; + process.env.TMPDIR = process.env.RENOVATE_TMPDIR || os.tmpdir(); + if (config.baseDir) { + logger.debug('Using configured baseDir: ' + config.baseDir); + } else { + config.baseDir = path.join(process.env.TMPDIR, 'renovate'); + logger.debug('Using baseDir: ' + config.baseDir); + } + await fs.ensureDir(config.baseDir); + if (config.cacheDir) { + logger.debug('Using configured cacheDir: ' + config.cacheDir); + } else { + config.cacheDir = path.join(config.baseDir, 'cache'); + logger.debug('Using cacheDir: ' + config.cacheDir); + } + await fs.ensureDir(config.cacheDir); + return config; +} + +export async function globalInitialize( + config_: RenovateConfig +): Promise { + let config = config_; + config = await initPlatform(config); + config = await setDirectories(config); + globalCache.init(config); + limits.init(config); + setEmojiConfig(config); + return config; +} + +export function globalFinalize(config: RenovateConfig): void { + globalCache.cleanup(config); +} From cb84fa9d9b5212e54364c1eba4437aa10e0da7f3 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Mon, 22 Jun 2020 20:18:39 +0200 Subject: [PATCH 03/25] fix: filter registryUrls for null/undefined --- lib/datasource/index.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/datasource/index.ts b/lib/datasource/index.ts index 2378c230198da3..68cd63b9512783 100644 --- a/lib/datasource/index.ts +++ b/lib/datasource/index.ts @@ -131,9 +131,14 @@ function resolveRegistryUrls( extractedUrls: string[] ): string[] { const { defaultRegistryUrls = [], appendRegistryUrls = [] } = datasource; - return is.nonEmptyArray(extractedUrls) - ? [...extractedUrls, ...appendRegistryUrls] - : [...defaultRegistryUrls, ...appendRegistryUrls]; + const customUrls = extractedUrls?.filter(Boolean); + let registryUrls: string[]; + if (is.nonEmptyArray(customUrls)) { + registryUrls = [...extractedUrls, ...appendRegistryUrls]; + } else { + registryUrls = [...defaultRegistryUrls, ...appendRegistryUrls]; + } + return registryUrls.filter(Boolean); } async function fetchReleases( From 52a074e0418ca160aa09ba9ba66bff7abafe45cb Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Mon, 22 Jun 2020 21:28:02 +0200 Subject: [PATCH 04/25] refactor: ExternalHostError (#6563) --- lib/config/presets/github/index.ts | 4 +-- .../gitlab/__snapshots__/index.spec.ts.snap | 2 +- lib/config/presets/gitlab/index.spec.ts | 6 ++--- lib/config/presets/gitlab/index.ts | 4 +-- lib/config/presets/index.ts | 12 +++------ lib/constants/error-messages.ts | 5 ++-- lib/datasource/cdnjs/index.spec.ts | 14 +++++----- lib/datasource/cdnjs/index.ts | 7 ++--- lib/datasource/common.ts | 17 ------------ .../crate/__snapshots__/index.spec.ts.snap | 2 +- lib/datasource/crate/index.ts | 10 +++---- .../dart/__snapshots__/index.spec.ts.snap | 2 +- lib/datasource/dart/index.ts | 5 ++-- lib/datasource/docker/index.spec.ts | 6 ++--- lib/datasource/docker/index.ts | 25 ++++++++--------- .../galaxy/__snapshots__/index.spec.ts.snap | 2 +- lib/datasource/galaxy/index.ts | 10 +++---- lib/datasource/gradle-version/index.ts | 5 ++-- .../helm/__snapshots__/index.spec.ts.snap | 2 +- lib/datasource/helm/index.ts | 5 ++-- lib/datasource/hex/index.spec.ts | 6 ++--- lib/datasource/hex/index.ts | 5 ++-- lib/datasource/index.spec.ts | 16 +++++------ lib/datasource/index.ts | 10 +++---- lib/datasource/maven/index.spec.ts | 6 ++--- lib/datasource/maven/util.ts | 4 +-- lib/datasource/npm/get.spec.ts | 4 +-- lib/datasource/npm/get.ts | 5 ++-- lib/datasource/npm/index.spec.ts | 6 ++--- lib/datasource/packagist/index.ts | 7 ++--- lib/datasource/pod/index.spec.ts | 3 ++- lib/datasource/pod/index.ts | 3 ++- lib/datasource/repology/index.spec.ts | 8 +++--- lib/datasource/repology/index.ts | 7 ++--- lib/datasource/ruby-version/index.ts | 5 ++-- lib/datasource/rubygems/get-rubygems-org.ts | 5 ++-- lib/datasource/terraform-module/index.ts | 5 ++-- lib/manager/gradle/index.ts | 6 ++--- lib/manager/npm/post-update/index.ts | 25 ++++++++++------- lib/manager/npm/post-update/yarn.ts | 5 ++-- lib/platform/git/storage.ts | 6 ++--- lib/platform/github/index.ts | 14 +++++----- lib/types/error.ts | 19 +++++++++++++ lib/util/http/github.spec.ts | 12 ++++----- lib/util/http/github.ts | 12 ++++----- lib/util/http/gitlab.ts | 8 +++--- lib/workers/branch/index.ts | 15 +++-------- lib/workers/pr/index.ts | 4 +-- lib/workers/repository/error.spec.ts | 20 +++++++------- lib/workers/repository/error.ts | 27 ++++++------------- lib/workers/repository/init/config.ts | 11 ++++---- 51 files changed, 211 insertions(+), 223 deletions(-) create mode 100644 lib/types/error.ts diff --git a/lib/config/presets/github/index.ts b/lib/config/presets/github/index.ts index 7363c3810dce02..2a781f073aea6e 100644 --- a/lib/config/presets/github/index.ts +++ b/lib/config/presets/github/index.ts @@ -1,6 +1,6 @@ -import { PLATFORM_FAILURE } from '../../../constants/error-messages'; import { PLATFORM_TYPE_GITHUB } from '../../../constants/platforms'; import { logger } from '../../../logger'; +import { ExternalHostError } from '../../../types/error'; import { Http, HttpOptions } from '../../../util/http'; import { Preset, PresetConfig } from '../common'; import { PRESET_DEP_NOT_FOUND, fetchPreset } from '../util'; @@ -27,7 +27,7 @@ export async function fetchJSONFile( res = await http.getJson(url, opts); } catch (err) { // istanbul ignore if: not testable with nock - if (err.message === PLATFORM_FAILURE) { + if (err instanceof ExternalHostError) { throw err; } logger.debug( diff --git a/lib/config/presets/gitlab/__snapshots__/index.spec.ts.snap b/lib/config/presets/gitlab/__snapshots__/index.spec.ts.snap index a04e3707f89e47..b79273bd383513 100644 --- a/lib/config/presets/gitlab/__snapshots__/index.spec.ts.snap +++ b/lib/config/presets/gitlab/__snapshots__/index.spec.ts.snap @@ -70,7 +70,7 @@ Array [ ] `; -exports[`config/presets/gitlab/index getPreset() throws platform-failure 1`] = ` +exports[`config/presets/gitlab/index getPreset() throws EXTERNAL_HOST_ERROR 1`] = ` Array [ Object { "headers": Object { diff --git a/lib/config/presets/gitlab/index.spec.ts b/lib/config/presets/gitlab/index.spec.ts index e180961dd3fb92..2d48a835a59c22 100644 --- a/lib/config/presets/gitlab/index.spec.ts +++ b/lib/config/presets/gitlab/index.spec.ts @@ -1,6 +1,6 @@ import * as httpMock from '../../../../test/httpMock'; import { getName } from '../../../../test/util'; -import { PLATFORM_FAILURE } from '../../../constants/error-messages'; +import { EXTERNAL_HOST_ERROR } from '../../../constants/error-messages'; import { PRESET_DEP_NOT_FOUND } from '../util'; import * as gitlab from '.'; @@ -18,14 +18,14 @@ describe(getName(__filename), () => { }); describe('getPreset()', () => { - it('throws platform-failure', async () => { + it('throws EXTERNAL_HOST_ERROR', async () => { httpMock.scope(gitlabApiHost).get(`${basePath}/branches`).reply(500); await expect( gitlab.getPreset({ packageName: 'some/repo', presetName: 'non-default', }) - ).rejects.toThrow(PLATFORM_FAILURE); + ).rejects.toThrow(EXTERNAL_HOST_ERROR); expect(httpMock.getTrace()).toMatchSnapshot(); }); diff --git a/lib/config/presets/gitlab/index.ts b/lib/config/presets/gitlab/index.ts index 7bec62cc2fff73..a5f7eb78211cf1 100644 --- a/lib/config/presets/gitlab/index.ts +++ b/lib/config/presets/gitlab/index.ts @@ -1,5 +1,5 @@ -import { PLATFORM_FAILURE } from '../../../constants/error-messages'; import { logger } from '../../../logger'; +import { ExternalHostError } from '../../../types/error'; import type { GitLabBranch } from '../../../types/platform/gitlab'; import { GitlabHttp } from '../../../util/http/gitlab'; import { Preset, PresetConfig } from '../common'; @@ -42,7 +42,7 @@ export async function fetchJSONFile( const url = `${endpoint}projects/${urlEncodedRepo}/repository/files/${urlEncodedPkgName}/raw?ref=${defautlBranchName}`; return (await gitlabApi.getJson(url)).body; } catch (err) { - if (err.message === PLATFORM_FAILURE) { + if (err instanceof ExternalHostError) { throw err; } logger.debug( diff --git a/lib/config/presets/index.ts b/lib/config/presets/index.ts index 144d068c958344..3a9eea8fdfe6d1 100644 --- a/lib/config/presets/index.ts +++ b/lib/config/presets/index.ts @@ -1,10 +1,7 @@ import is from '@sindresorhus/is'; -import { - CONFIG_VALIDATION, - DATASOURCE_FAILURE, - PLATFORM_FAILURE, -} from '../../constants/error-messages'; +import { CONFIG_VALIDATION } from '../../constants/error-messages'; import { logger } from '../../logger'; +import { ExternalHostError } from '../../types/error'; import { regEx } from '../../util/regex'; import { RenovateConfig } from '../common'; import * as massage from '../massage'; @@ -207,10 +204,7 @@ export async function resolveConfigPresets( } catch (err) { logger.debug({ preset, err }, 'Preset fetch error'); // istanbul ignore if - if ( - err.message === PLATFORM_FAILURE || - err.message === DATASOURCE_FAILURE - ) { + if (err instanceof ExternalHostError) { throw err; } const error = new Error(CONFIG_VALIDATION); diff --git a/lib/constants/error-messages.ts b/lib/constants/error-messages.ts index b1bb53ef6de252..12ad1164c19456 100644 --- a/lib/constants/error-messages.ts +++ b/lib/constants/error-messages.ts @@ -5,7 +5,6 @@ export const SYSTEM_INSUFFICIENT_MEMORY = 'out-of-memory'; // Platform Error export const PLATFORM_AUTHENTICATION_ERROR = 'authentication-error'; export const PLATFORM_BAD_CREDENTIALS = 'bad-credentials'; -export const PLATFORM_FAILURE = 'platform-failure'; export const PLATFORM_GPG_FAILED = 'gpg-failed'; export const PLATFORM_INTEGRATION_UNAUTHORIZED = 'integration-unauthorized'; export const PLATFORM_NOT_FOUND = 'platform-not-found'; @@ -34,8 +33,8 @@ export const REPOSITORY_UNINITIATED = 'uninitiated'; export const MANAGER_LOCKFILE_ERROR = 'lockfile-error'; export const MANAGER_NO_PACKAGE_FILES = 'no-package-files'; -// Datasource error -export const DATASOURCE_FAILURE = 'registry-failure'; +// Host error +export const EXTERNAL_HOST_ERROR = 'external-host-error'; // Worker Error export const WORKER_FILE_UPDATE_FAILED = 'update-failure'; diff --git a/lib/datasource/cdnjs/index.spec.ts b/lib/datasource/cdnjs/index.spec.ts index 976b61003caad9..2ff92f7cbf5e6d 100644 --- a/lib/datasource/cdnjs/index.spec.ts +++ b/lib/datasource/cdnjs/index.spec.ts @@ -1,7 +1,7 @@ import fs from 'fs'; import { getPkgReleases } from '..'; import * as httpMock from '../../../test/httpMock'; -import { DATASOURCE_FAILURE } from '../../constants/error-messages'; +import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages'; import { id as datasource } from '.'; let res1 = fs.readFileSync( @@ -36,14 +36,14 @@ describe('datasource/cdnjs', () => { httpMock.scope(baseUrl).get(pathFor('foo/bar')).reply(200, null); await expect( getPkgReleases({ datasource, depName: 'foo/bar' }) - ).rejects.toThrow(DATASOURCE_FAILURE); + ).rejects.toThrow(EXTERNAL_HOST_ERROR); expect(httpMock.getTrace()).toMatchSnapshot(); }); it('throws for error', async () => { httpMock.scope(baseUrl).get(pathFor('foo/bar')).replyWithError('error'); await expect( getPkgReleases({ datasource, depName: 'foo/bar' }) - ).rejects.toThrow(DATASOURCE_FAILURE); + ).rejects.toThrow(EXTERNAL_HOST_ERROR); expect(httpMock.getTrace()).toMatchSnapshot(); }); it('returns null for 404', async () => { @@ -70,28 +70,28 @@ describe('datasource/cdnjs', () => { httpMock.scope(baseUrl).get(pathFor('foo/bar')).reply(401); await expect( getPkgReleases({ datasource, depName: 'foo/bar' }) - ).rejects.toThrow(DATASOURCE_FAILURE); + ).rejects.toThrow(EXTERNAL_HOST_ERROR); expect(httpMock.getTrace()).toMatchSnapshot(); }); it('throws for 429', async () => { httpMock.scope(baseUrl).get(pathFor('foo/bar')).reply(429); await expect( getPkgReleases({ datasource, depName: 'foo/bar' }) - ).rejects.toThrow(DATASOURCE_FAILURE); + ).rejects.toThrow(EXTERNAL_HOST_ERROR); expect(httpMock.getTrace()).toMatchSnapshot(); }); it('throws for 5xx', async () => { httpMock.scope(baseUrl).get(pathFor('foo/bar')).reply(502); await expect( getPkgReleases({ datasource, depName: 'foo/bar' }) - ).rejects.toThrow(DATASOURCE_FAILURE); + ).rejects.toThrow(EXTERNAL_HOST_ERROR); expect(httpMock.getTrace()).toMatchSnapshot(); }); it('returns null for unknown error', async () => { httpMock.scope(baseUrl).get(pathFor('foo/bar')).replyWithError('error'); await expect( getPkgReleases({ datasource, depName: 'foo/bar' }) - ).rejects.toThrow(DATASOURCE_FAILURE); + ).rejects.toThrow(EXTERNAL_HOST_ERROR); expect(httpMock.getTrace()).toMatchSnapshot(); }); it('processes real data', async () => { diff --git a/lib/datasource/cdnjs/index.ts b/lib/datasource/cdnjs/index.ts index 5b5f7f20264a72..c408bd0d147e71 100644 --- a/lib/datasource/cdnjs/index.ts +++ b/lib/datasource/cdnjs/index.ts @@ -1,7 +1,8 @@ import { logger } from '../../logger'; +import { ExternalHostError } from '../../types/error'; import { Http } from '../../util/http'; import { CachePromise, cacheAble } from '../cache'; -import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common'; +import { GetReleasesConfig, ReleaseResult } from '../common'; export const id = 'cdnjs'; @@ -60,7 +61,7 @@ export async function getReleases({ logger.debug({ library }, 'cdnjs library not found'); return null; } - // Throw a DatasourceError for all other types of errors - throw new DatasourceError(err); + // Throw an ExternalHostError for all other types of errors + throw new ExternalHostError(err); } } diff --git a/lib/datasource/common.ts b/lib/datasource/common.ts index 1d4be8b8c4d349..0f31ca9cb227ee 100644 --- a/lib/datasource/common.ts +++ b/lib/datasource/common.ts @@ -1,5 +1,3 @@ -import { DATASOURCE_FAILURE } from '../constants/error-messages'; - export interface Config { datasource?: string; depName?: string; @@ -80,18 +78,3 @@ export interface Datasource { defaultConfig?: object; registryStrategy?: 'first' | 'hunt' | 'merge'; } - -export class DatasourceError extends Error { - err: Error; - - datasource?: string; - - lookupName?: string; - - constructor(err: Error) { - super(DATASOURCE_FAILURE); - // Set the prototype explicitly: https://github.com/Microsoft/TypeScript/wiki/Breaking-Changes#extending-built-ins-like-error-array-and-map-may-no-longer-work - Object.setPrototypeOf(this, DatasourceError.prototype); - this.err = err; - } -} diff --git a/lib/datasource/crate/__snapshots__/index.spec.ts.snap b/lib/datasource/crate/__snapshots__/index.spec.ts.snap index 63116b90b90601..f5d2a6e7250c73 100644 --- a/lib/datasource/crate/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/crate/__snapshots__/index.spec.ts.snap @@ -764,7 +764,7 @@ Array [ ] `; -exports[`datasource/crate getReleases throws for 5xx 1`] = `[Error: registry-failure]`; +exports[`datasource/crate getReleases throws for 5xx 1`] = `[Error: external-host-error]`; exports[`datasource/crate getReleases throws for 5xx 2`] = ` Array [ diff --git a/lib/datasource/crate/index.ts b/lib/datasource/crate/index.ts index fe948c1ab79079..40134c936da8ca 100644 --- a/lib/datasource/crate/index.ts +++ b/lib/datasource/crate/index.ts @@ -1,12 +1,8 @@ import { logger } from '../../logger'; +import { ExternalHostError } from '../../types/error'; import * as globalCache from '../../util/cache/global'; import { Http } from '../../util/http'; -import { - DatasourceError, - GetReleasesConfig, - Release, - ReleaseResult, -} from '../common'; +import { GetReleasesConfig, Release, ReleaseResult } from '../common'; export const id = 'crate'; @@ -105,7 +101,7 @@ export async function getReleases({ err.statusCode === 429 || (err.statusCode >= 500 && err.statusCode < 600) ) { - throw new DatasourceError(err); + throw new ExternalHostError(err); } logger.warn({ err, lookupName }, 'crates.io lookup failure: Unknown error'); return null; diff --git a/lib/datasource/dart/__snapshots__/index.spec.ts.snap b/lib/datasource/dart/__snapshots__/index.spec.ts.snap index 39f878f58fd9a7..856d329c5d3f78 100644 --- a/lib/datasource/dart/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/dart/__snapshots__/index.spec.ts.snap @@ -144,7 +144,7 @@ Array [ ] `; -exports[`datasource/dart getReleases throws for 5xx 1`] = `[Error: registry-failure]`; +exports[`datasource/dart getReleases throws for 5xx 1`] = `[Error: external-host-error]`; exports[`datasource/dart getReleases throws for 5xx 2`] = ` Array [ diff --git a/lib/datasource/dart/index.ts b/lib/datasource/dart/index.ts index e080ef053f3b51..4043f1df5b3248 100644 --- a/lib/datasource/dart/index.ts +++ b/lib/datasource/dart/index.ts @@ -1,6 +1,7 @@ import { logger } from '../../logger'; +import { ExternalHostError } from '../../types/error'; import { Http, HttpResponse } from '../../util/http'; -import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common'; +import { GetReleasesConfig, ReleaseResult } from '../common'; export const id = 'dart'; @@ -33,7 +34,7 @@ export async function getReleases({ err.statusCode === 429 || (err.statusCode >= 500 && err.statusCode < 600) ) { - throw new DatasourceError(err); + throw new ExternalHostError(err); } logger.warn( { err, lookupName }, diff --git a/lib/datasource/docker/index.spec.ts b/lib/datasource/docker/index.spec.ts index 7d2c09c7ffe7d4..2f69ffa7d32564 100644 --- a/lib/datasource/docker/index.spec.ts +++ b/lib/datasource/docker/index.spec.ts @@ -2,7 +2,7 @@ import AWS from 'aws-sdk'; import AWSMock from 'aws-sdk-mock'; import { getDigest, getPkgReleases } from '..'; import * as httpMock from '../../../test/httpMock'; -import { DATASOURCE_FAILURE } from '../../constants/error-messages'; +import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages'; import * as _hostRules from '../../util/host-rules'; import * as docker from '.'; @@ -331,13 +331,13 @@ describe('api/docker', () => { httpMock.scope(baseUrl).get('/').replyWithError({ statusCode: 429 }); await expect( getDigest({ datasource: 'docker', depName: 'some-dep' }, 'latest') - ).rejects.toThrow(Error(DATASOURCE_FAILURE)); + ).rejects.toThrow(EXTERNAL_HOST_ERROR); }); it('should throw error for 5xx', async () => { httpMock.scope(baseUrl).get('/').replyWithError({ statusCode: 504 }); await expect( getDigest({ datasource: 'docker', depName: 'some-dep' }, 'latest') - ).rejects.toThrow(Error(DATASOURCE_FAILURE)); + ).rejects.toThrow(EXTERNAL_HOST_ERROR); }); }); describe('getReleases', () => { diff --git a/lib/datasource/docker/index.ts b/lib/datasource/docker/index.ts index 34536479d3370f..7a9e57c4199f41 100644 --- a/lib/datasource/docker/index.ts +++ b/lib/datasource/docker/index.ts @@ -6,10 +6,11 @@ import parseLinkHeader from 'parse-link-header'; import wwwAuthenticate from 'www-authenticate'; import { logger } from '../../logger'; import { HostRule } from '../../types'; +import { ExternalHostError } from '../../types/error'; import * as globalCache from '../../util/cache/global'; import * as hostRules from '../../util/host-rules'; import { Http, HttpResponse } from '../../util/http'; -import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common'; +import { GetReleasesConfig, ReleaseResult } from '../common'; // TODO: add got typings when available // TODO: replace www-authenticate with https://www.npmjs.com/package/auth-header ? @@ -220,14 +221,14 @@ async function getAuthHeaders( } // prettier-ignore if (err.name === 'RequestError' && registry.endsWith('docker.io')) { // lgtm [js/incomplete-url-substring-sanitization] - throw new DatasourceError(err); + throw new ExternalHostError(err); } // prettier-ignore if (err.statusCode === 429 && registry.endsWith('docker.io')) { // lgtm [js/incomplete-url-substring-sanitization] - throw new DatasourceError(err); + throw new ExternalHostError(err); } if (err.statusCode >= 500 && err.statusCode < 600) { - throw new DatasourceError(err); + throw new ExternalHostError(err); } logger.warn( { registry, dockerRepository: repository, err }, @@ -267,7 +268,7 @@ async function getManifestResponse( }); return manifestResponse; } catch (err) /* istanbul ignore next */ { - if (err instanceof DatasourceError) { + if (err instanceof ExternalHostError) { throw err; } if (err.statusCode === 401) { @@ -292,10 +293,10 @@ async function getManifestResponse( } // prettier-ignore if (err.statusCode === 429 && registry.endsWith('docker.io')) { // lgtm [js/incomplete-url-substring-sanitization] - throw new DatasourceError(err); + throw new ExternalHostError(err); } if (err.statusCode >= 500 && err.statusCode < 600) { - throw new DatasourceError(err); + throw new ExternalHostError(err); } if (err.code === 'ETIMEDOUT') { logger.debug( @@ -356,7 +357,7 @@ export async function getDigest( logger.debug({ digest }, 'Got docker digest'); } } catch (err) /* istanbul ignore next */ { - if (err instanceof DatasourceError) { + if (err instanceof ExternalHostError) { throw err; } logger.debug( @@ -413,7 +414,7 @@ async function getTags( await globalCache.set(cacheNamespace, cacheKey, tags, cacheMinutes); return tags; } catch (err) /* istanbul ignore next */ { - if (err instanceof DatasourceError) { + if (err instanceof ExternalHostError) { throw err; } logger.debug( @@ -441,14 +442,14 @@ async function getTags( { registry, dockerRepository: repository, err }, 'docker registry failure: too many requests' ); - throw new DatasourceError(err); + throw new ExternalHostError(err); } if (err.statusCode >= 500 && err.statusCode < 600) { logger.warn( { registry, dockerRepository: repository, err }, 'docker registry failure: internal error' ); - throw new DatasourceError(err); + throw new ExternalHostError(err); } if (err.code === 'ETIMEDOUT') { logger.debug( @@ -543,7 +544,7 @@ async function getLabels( await globalCache.set(cacheNamespace, cacheKey, labels, cacheMinutes); return labels; } catch (err) { - if (err instanceof DatasourceError) { + if (err instanceof ExternalHostError) { throw err; } if (err.statusCode === 400 || err.statusCode === 401) { diff --git a/lib/datasource/galaxy/__snapshots__/index.spec.ts.snap b/lib/datasource/galaxy/__snapshots__/index.spec.ts.snap index 1ecdb5c221d5cd..3e2f68cc8b7ee2 100644 --- a/lib/datasource/galaxy/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/galaxy/__snapshots__/index.spec.ts.snap @@ -119,7 +119,7 @@ Array [ ] `; -exports[`datasource/galaxy getReleases throws for 5xx 1`] = `[Error: registry-failure]`; +exports[`datasource/galaxy getReleases throws for 5xx 1`] = `[Error: external-host-error]`; exports[`datasource/galaxy getReleases throws for 5xx 2`] = ` Array [ diff --git a/lib/datasource/galaxy/index.ts b/lib/datasource/galaxy/index.ts index ce6b9e89643596..a76942dfd4c43f 100644 --- a/lib/datasource/galaxy/index.ts +++ b/lib/datasource/galaxy/index.ts @@ -1,12 +1,8 @@ import { logger } from '../../logger'; +import { ExternalHostError } from '../../types/error'; import * as globalCache from '../../util/cache/global'; import { Http } from '../../util/http'; -import { - DatasourceError, - GetReleasesConfig, - Release, - ReleaseResult, -} from '../common'; +import { GetReleasesConfig, Release, ReleaseResult } from '../common'; export const id = 'galaxy'; @@ -101,7 +97,7 @@ export async function getReleases({ err.statusCode === 429 || (err.statusCode >= 500 && err.statusCode < 600) ) { - throw new DatasourceError(err); + throw new ExternalHostError(err); } return null; } diff --git a/lib/datasource/gradle-version/index.ts b/lib/datasource/gradle-version/index.ts index aab1d9d61399d5..624bb27b5ea89e 100644 --- a/lib/datasource/gradle-version/index.ts +++ b/lib/datasource/gradle-version/index.ts @@ -1,7 +1,8 @@ import { logger } from '../../logger'; +import { ExternalHostError } from '../../types/error'; import { Http } from '../../util/http'; import { regEx } from '../../util/regex'; -import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common'; +import { GetReleasesConfig, ReleaseResult } from '../common'; export const id = 'gradle-version'; export const defaultRegistryUrls = ['https://services.gradle.org/versions/all']; @@ -50,7 +51,7 @@ export async function getReleases({ })); } catch (err) /* istanbul ignore next */ { if (err.host === 'services.gradle.org') { - throw new DatasourceError(err); + throw new ExternalHostError(err); } logger.debug({ err }, 'gradle-version err'); return null; diff --git a/lib/datasource/helm/__snapshots__/index.spec.ts.snap b/lib/datasource/helm/__snapshots__/index.spec.ts.snap index 8ae2aea4bbf36f..d348280ab66f46 100644 --- a/lib/datasource/helm/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/helm/__snapshots__/index.spec.ts.snap @@ -244,7 +244,7 @@ Array [ ] `; -exports[`datasource/helm getReleases throws for 5xx 1`] = `[Error: registry-failure]`; +exports[`datasource/helm getReleases throws for 5xx 1`] = `[Error: external-host-error]`; exports[`datasource/helm getReleases throws for 5xx 2`] = ` Array [ diff --git a/lib/datasource/helm/index.ts b/lib/datasource/helm/index.ts index 4b684275219c16..5ba8ba238fa10e 100644 --- a/lib/datasource/helm/index.ts +++ b/lib/datasource/helm/index.ts @@ -1,10 +1,11 @@ import yaml from 'js-yaml'; import { logger } from '../../logger'; +import { ExternalHostError } from '../../types/error'; import * as globalCache from '../../util/cache/global'; import { Http } from '../../util/http'; import { ensureTrailingSlash } from '../../util/url'; -import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common'; +import { GetReleasesConfig, ReleaseResult } from '../common'; export const id = 'helm'; @@ -60,7 +61,7 @@ export async function getRepositoryData( err.statusCode === 429 || (err.statusCode >= 500 && err.statusCode < 600) ) { - throw new DatasourceError(err); + throw new ExternalHostError(err); } // istanbul ignore if if (err.name === 'UnsupportedProtocolError') { diff --git a/lib/datasource/hex/index.spec.ts b/lib/datasource/hex/index.spec.ts index a751a20da5061b..14dabf83f2dfda 100644 --- a/lib/datasource/hex/index.spec.ts +++ b/lib/datasource/hex/index.spec.ts @@ -1,7 +1,7 @@ import fs from 'fs'; import { getPkgReleases } from '..'; import * as httpMock from '../../../test/httpMock'; -import { DATASOURCE_FAILURE } from '../../constants/error-messages'; +import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages'; import * as _hostRules from '../../util/host-rules'; import { id as datasource } from '.'; @@ -67,14 +67,14 @@ describe('datasource/hex', () => { httpMock.scope(baseUrl).get('/some_crate').reply(429); await expect( getPkgReleases({ datasource, depName: 'some_crate' }) - ).rejects.toThrow(DATASOURCE_FAILURE); + ).rejects.toThrow(EXTERNAL_HOST_ERROR); expect(httpMock.getTrace()).toMatchSnapshot(); }); it('throws for 5xx', async () => { httpMock.scope(baseUrl).get('/some_crate').reply(502); await expect( getPkgReleases({ datasource, depName: 'some_crate' }) - ).rejects.toThrow(DATASOURCE_FAILURE); + ).rejects.toThrow(EXTERNAL_HOST_ERROR); expect(httpMock.getTrace()).toMatchSnapshot(); }); it('returns null for unknown error', async () => { diff --git a/lib/datasource/hex/index.ts b/lib/datasource/hex/index.ts index dc614c97ee1897..c619cbffc0ba13 100644 --- a/lib/datasource/hex/index.ts +++ b/lib/datasource/hex/index.ts @@ -1,6 +1,7 @@ import { logger } from '../../logger'; +import { ExternalHostError } from '../../types/error'; import { Http } from '../../util/http'; -import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common'; +import { GetReleasesConfig, ReleaseResult } from '../common'; export const id = 'hex'; @@ -69,7 +70,7 @@ export async function getReleases({ err.statusCode === 429 || (err.statusCode >= 500 && err.statusCode < 600) ) { - throw new DatasourceError(err); + throw new ExternalHostError(err); } if (err.statusCode === 401) { diff --git a/lib/datasource/index.spec.ts b/lib/datasource/index.spec.ts index 974003a4b0e4e9..34aa6531a7a9bc 100644 --- a/lib/datasource/index.spec.ts +++ b/lib/datasource/index.spec.ts @@ -1,7 +1,7 @@ import { mocked } from '../../test/util'; -import { DATASOURCE_FAILURE } from '../constants/error-messages'; +import { EXTERNAL_HOST_ERROR } from '../constants/error-messages'; +import { ExternalHostError } from '../types/error'; import { loadModules } from '../util/modules'; -import { DatasourceError } from './common'; import * as datasourceDocker from './docker'; import * as datasourceGithubTags from './github-tags'; import * as datasourceMaven from './maven'; @@ -131,9 +131,9 @@ describe('datasource/index', () => { }); expect(res).not.toBeNull(); }); - it('hunts registries and aborts on DatasourceError', async () => { + it('hunts registries and aborts on ExternalHostError', async () => { packagistDatasource.getReleases.mockImplementationOnce(() => { - throw new DatasourceError(new Error()); + throw new ExternalHostError(new Error()); }); await expect( datasource.getPkgReleases({ @@ -141,7 +141,7 @@ describe('datasource/index', () => { depName: 'something', registryUrls: ['https://reg1.com', 'https://reg2.io'], }) - ).rejects.toThrow(DATASOURCE_FAILURE); + ).rejects.toThrow(EXTERNAL_HOST_ERROR); }); it('hunts registries and passes on error', async () => { packagistDatasource.getReleases.mockImplementationOnce(() => { @@ -173,9 +173,9 @@ describe('datasource/index', () => { expect(res).toMatchSnapshot(); expect(res.releases).toHaveLength(2); }); - it('merges registries and aborts on DatasourceError', async () => { + it('merges registries and aborts on ExternalHostError', async () => { mavenDatasource.getReleases.mockImplementationOnce(() => { - throw new DatasourceError(new Error()); + throw new ExternalHostError(new Error()); }); await expect( datasource.getPkgReleases({ @@ -183,7 +183,7 @@ describe('datasource/index', () => { depName: 'something', registryUrls: ['https://reg1.com', 'https://reg2.io'], }) - ).rejects.toThrow(DATASOURCE_FAILURE); + ).rejects.toThrow(EXTERNAL_HOST_ERROR); }); it('merges registries and passes on error', async () => { mavenDatasource.getReleases.mockImplementationOnce(() => { diff --git a/lib/datasource/index.ts b/lib/datasource/index.ts index 68cd63b9512783..dba7f32e5fef06 100644 --- a/lib/datasource/index.ts +++ b/lib/datasource/index.ts @@ -1,13 +1,13 @@ import is from '@sindresorhus/is'; import _ from 'lodash'; import { logger } from '../logger'; +import { ExternalHostError } from '../types/error'; import * as runCache from '../util/cache/run'; import { clone } from '../util/clone'; import * as allVersioning from '../versioning'; import datasources from './api.generated'; import { Datasource, - DatasourceError, DigestConfig, GetPkgReleasesConfig, GetReleasesConfig, @@ -65,7 +65,7 @@ async function huntRegistries( break; } } catch (err) { - if (err instanceof DatasourceError) { + if (err instanceof ExternalHostError) { throw err; } // We'll always save the last-thrown error @@ -100,7 +100,7 @@ async function mergeRegistries( combinedRes = res; } } catch (err) { - if (err instanceof DatasourceError) { + if (err instanceof ExternalHostError) { throw err; } // We'll always save the last-thrown error @@ -221,8 +221,8 @@ export async function getPkgReleases( }) ); } catch (e) /* istanbul ignore next */ { - if (e instanceof DatasourceError) { - e.datasource = config.datasource; + if (e instanceof ExternalHostError) { + e.hostType = config.datasource; e.lookupName = lookupName; } throw e; diff --git a/lib/datasource/maven/index.spec.ts b/lib/datasource/maven/index.spec.ts index bbe3d11b6d0709..4349d4ba29f149 100644 --- a/lib/datasource/maven/index.spec.ts +++ b/lib/datasource/maven/index.spec.ts @@ -2,7 +2,7 @@ import fs from 'fs'; import { resolve } from 'path'; import nock from 'nock'; import { getPkgReleases } from '..'; -import { DATASOURCE_FAILURE } from '../../constants/error-messages'; +import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages'; import * as hostRules from '../../util/host-rules'; import * as mavenVersioning from '../../versioning/maven'; import { id as datasource } from '.'; @@ -182,7 +182,7 @@ describe('datasource/maven', () => { expect(releases.releases).toEqual(generateReleases(MYSQL_VERSIONS)); }); - it('should throw registry-failure if default maven repo fails', async () => { + it('should throw external-host-error if default maven repo fails', async () => { nock('https://repo.maven.apache.org') .get('/maven2/org/artifact/maven-metadata.xml') .times(4) @@ -195,7 +195,7 @@ describe('datasource/maven', () => { depName: 'org:artifact', registryUrls: ['https://repo.maven.apache.org/maven2/'], }) - ).rejects.toThrow(Error(DATASOURCE_FAILURE)); + ).rejects.toThrow(EXTERNAL_HOST_ERROR); }); it('should return all versions of a specific library if a repository fails because invalid protocol', async () => { diff --git a/lib/datasource/maven/util.ts b/lib/datasource/maven/util.ts index 4639aea1c39302..bb03867e9a390f 100644 --- a/lib/datasource/maven/util.ts +++ b/lib/datasource/maven/util.ts @@ -1,7 +1,7 @@ import url from 'url'; import { logger } from '../../logger'; +import { ExternalHostError } from '../../types/error'; import { Http } from '../../util/http'; -import { DatasourceError } from '../common'; import { MAVEN_REPO, id } from './common'; @@ -77,7 +77,7 @@ export async function downloadHttpProtocol( } else if (isTemporalError(err)) { logger.debug({ failedUrl, err }, 'Temporary error'); if (isMavenCentral(pkgUrl)) { - throw new DatasourceError(err); + throw new ExternalHostError(err); } } else if (isConnectionError(err)) { // istanbul ignore next diff --git a/lib/datasource/npm/get.spec.ts b/lib/datasource/npm/get.spec.ts index 2d394b010d4ade..ec305f257982ec 100644 --- a/lib/datasource/npm/get.spec.ts +++ b/lib/datasource/npm/get.spec.ts @@ -1,6 +1,6 @@ import * as httpMock from '../../../test/httpMock'; import { getName } from '../../../test/util'; -import { DatasourceError } from '../common'; +import { ExternalHostError } from '../../types/error'; import { getDependency, resetMemCache } from './get'; import { setNpmrc } from './npmrc'; @@ -150,7 +150,7 @@ describe(getName(__filename), () => { .get('/npm-parse-error') .reply(200, 'not-a-json'); await expect(getDependency('npm-parse-error', 0)).rejects.toThrow( - DatasourceError + ExternalHostError ); httpMock diff --git a/lib/datasource/npm/get.ts b/lib/datasource/npm/get.ts index d2f0e2ab362cc6..dee919732af7e7 100644 --- a/lib/datasource/npm/get.ts +++ b/lib/datasource/npm/get.ts @@ -6,11 +6,12 @@ import moment from 'moment'; import registryAuthToken from 'registry-auth-token'; import getRegistryUrl from 'registry-auth-token/registry-url'; import { logger } from '../../logger'; +import { ExternalHostError } from '../../types/error'; import * as globalCache from '../../util/cache/global'; import { find } from '../../util/host-rules'; import { Http, HttpOptions } from '../../util/http'; import { maskToken } from '../../util/mask'; -import { DatasourceError, Release, ReleaseResult } from '../common'; +import { Release, ReleaseResult } from '../common'; import { id } from './common'; import { getNpmrc } from './npmrc'; @@ -280,7 +281,7 @@ export async function getDependency( if (err.name === 'ParseError' && err.body) { err.body = 'err.body deleted by Renovate'; } - throw new DatasourceError(err); + throw new ExternalHostError(err); } return null; } diff --git a/lib/datasource/npm/index.spec.ts b/lib/datasource/npm/index.spec.ts index f2106aacbc6c1e..00dfde3e164bce 100644 --- a/lib/datasource/npm/index.spec.ts +++ b/lib/datasource/npm/index.spec.ts @@ -3,7 +3,7 @@ import nock from 'nock'; import _registryAuthToken from 'registry-auth-token'; import { getPkgReleases } from '..'; import { getName } from '../../../test/util'; -import { DATASOURCE_FAILURE } from '../../constants/error-messages'; +import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages'; import * as hostRules from '../../util/host-rules'; import { id as datasource, getNpmrc, resetCache, setNpmrc } from '.'; @@ -204,13 +204,13 @@ describe(getName(__filename), () => { nock('https://registry.npmjs.org').get('/foobar').reply(503); await expect( getPkgReleases({ datasource, depName: 'foobar' }) - ).rejects.toThrow(Error(DATASOURCE_FAILURE)); + ).rejects.toThrow(EXTERNAL_HOST_ERROR); }); it('should throw error for 408', async () => { nock('https://registry.npmjs.org').get('/foobar').reply(408); await expect( getPkgReleases({ datasource, depName: 'foobar' }) - ).rejects.toThrow(Error(DATASOURCE_FAILURE)); + ).rejects.toThrow(EXTERNAL_HOST_ERROR); }); it('should throw error for others', async () => { nock('https://registry.npmjs.org').get('/foobar').reply(451); diff --git a/lib/datasource/packagist/index.ts b/lib/datasource/packagist/index.ts index 77946c2bb4cb3d..7fce259e40b1d5 100644 --- a/lib/datasource/packagist/index.ts +++ b/lib/datasource/packagist/index.ts @@ -2,11 +2,12 @@ import URL from 'url'; import pAll from 'p-all'; import { logger } from '../../logger'; +import { ExternalHostError } from '../../types/error'; import * as globalCache from '../../util/cache/global'; import * as runCache from '../../util/cache/run'; import * as hostRules from '../../util/host-rules'; import { Http, HttpOptions } from '../../util/http'; -import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common'; +import { GetReleasesConfig, ReleaseResult } from '../common'; export const id = 'packagist'; export const defaultRegistryUrls = ['https://packagist.org']; @@ -313,10 +314,10 @@ async function packageLookup( } if (err.host === 'packagist.org') { if (err.code === 'ECONNRESET' || err.code === 'ETIMEDOUT') { - throw new DatasourceError(err); + throw new ExternalHostError(err); } if (err.statusCode && err.statusCode >= 500 && err.statusCode < 600) { - throw new DatasourceError(err); + throw new ExternalHostError(err); } } logger.warn({ err, name }, 'packagist registry failure: Unknown error'); diff --git a/lib/datasource/pod/index.spec.ts b/lib/datasource/pod/index.spec.ts index 7a637e56befb68..780f51d935b4d6 100644 --- a/lib/datasource/pod/index.spec.ts +++ b/lib/datasource/pod/index.spec.ts @@ -1,5 +1,6 @@ import { getPkgReleases } from '..'; import * as httpMock from '../../../test/httpMock'; +import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages'; import * as rubyVersioning from '../../versioning/ruby'; import * as pod from '.'; @@ -63,7 +64,7 @@ describe('datasource/cocoapods', () => { .scope(cocoapodsHost) .get('/all_pods_versions_a_c_b.txt') .reply(429); - await expect(getPkgReleases(config)).rejects.toThrow('registry-failure'); + await expect(getPkgReleases(config)).rejects.toThrow(EXTERNAL_HOST_ERROR); expect(httpMock.getTrace()).toMatchSnapshot(); }); it('returns null for unknown error', async () => { diff --git a/lib/datasource/pod/index.ts b/lib/datasource/pod/index.ts index 24b0361f875175..391719f73187cb 100644 --- a/lib/datasource/pod/index.ts +++ b/lib/datasource/pod/index.ts @@ -1,5 +1,6 @@ import crypto from 'crypto'; import { logger } from '../../logger'; +import { ExternalHostError } from '../../types/error'; import * as globalCache from '../../util/cache/global'; import { Http } from '../../util/http'; import { GithubHttp } from '../../util/http/github'; @@ -44,7 +45,7 @@ function handleError(lookupName: string, err: Error): void { (err.statusCode >= 500 && err.statusCode < 600) ) { logger.warn({ lookupName, err }, `CocoaPods registry failure`); - throw new Error('registry-failure'); + throw new ExternalHostError(err); } if (err.statusCode === 401) { diff --git a/lib/datasource/repology/index.spec.ts b/lib/datasource/repology/index.spec.ts index 56b34cdcab7086..bd08b9d04eb223 100644 --- a/lib/datasource/repology/index.spec.ts +++ b/lib/datasource/repology/index.spec.ts @@ -2,7 +2,7 @@ import fs from 'fs'; import { getPkgReleases } from '..'; import * as httpMock from '../../../test/httpMock'; import { getName } from '../../../test/util'; -import { DATASOURCE_FAILURE } from '../../constants/error-messages'; +import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages'; import { id as versioning } from '../../versioning/loose'; import { RepologyPackage, id as datasource } from '.'; @@ -127,7 +127,7 @@ describe(getName(__filename), () => { versioning, depName: 'debian_stable/nginx', }) - ).rejects.toThrow(DATASOURCE_FAILURE); + ).rejects.toThrow(EXTERNAL_HOST_ERROR); expect(httpMock.getTrace()).toMatchSnapshot(); }); @@ -140,7 +140,7 @@ describe(getName(__filename), () => { versioning, depName: 'debian_stable/nginx', }) - ).rejects.toThrow(DATASOURCE_FAILURE); + ).rejects.toThrow(EXTERNAL_HOST_ERROR); expect(httpMock.getTrace()).toMatchSnapshot(); }); @@ -151,7 +151,7 @@ describe(getName(__filename), () => { versioning, depName: 'invalid-lookup-name', }) - ).rejects.toThrow(DATASOURCE_FAILURE); + ).rejects.toThrow(EXTERNAL_HOST_ERROR); expect(httpMock.getTrace()).toMatchSnapshot(); }); diff --git a/lib/datasource/repology/index.ts b/lib/datasource/repology/index.ts index e45c10f021ed23..7af17eb7753cff 100644 --- a/lib/datasource/repology/index.ts +++ b/lib/datasource/repology/index.ts @@ -1,8 +1,9 @@ import { URLSearchParams } from 'url'; import { logger } from '../../logger'; +import { ExternalHostError } from '../../types/error'; import * as globalCache from '../../util/cache/global'; import { Http } from '../../util/http'; -import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common'; +import { GetReleasesConfig, ReleaseResult } from '../common'; export const id = 'repology'; @@ -117,7 +118,7 @@ export async function getReleases({ // Ensure lookup name contains both repository and package const [repoName, pkgName] = lookupName.split('/', 2); if (!repoName || !pkgName) { - throw new DatasourceError( + throw new ExternalHostError( new Error( 'Repology lookup name must contain repository and package separated by slash (/)' ) @@ -142,6 +143,6 @@ export async function getReleases({ { lookupName, err }, 'Repology lookup failed with unexpected error' ); - throw new DatasourceError(err); + throw new ExternalHostError(err); } } diff --git a/lib/datasource/ruby-version/index.ts b/lib/datasource/ruby-version/index.ts index c3315f04e16dc2..d85ee6cec9d591 100644 --- a/lib/datasource/ruby-version/index.ts +++ b/lib/datasource/ruby-version/index.ts @@ -1,9 +1,10 @@ import { parse } from 'node-html-parser'; +import { ExternalHostError } from '../../types/error'; import * as globalCache from '../../util/cache/global'; import { Http } from '../../util/http'; import { isVersion } from '../../versioning/ruby'; -import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common'; +import { GetReleasesConfig, ReleaseResult } from '../common'; export const id = 'ruby-version'; @@ -51,6 +52,6 @@ export async function getReleases( await globalCache.set(cacheNamespace, 'all', res, 15); return res; } catch (err) { - throw new DatasourceError(err); + throw new ExternalHostError(err); } } diff --git a/lib/datasource/rubygems/get-rubygems-org.ts b/lib/datasource/rubygems/get-rubygems-org.ts index 3eb01a4ed97a88..f3005573227fa5 100644 --- a/lib/datasource/rubygems/get-rubygems-org.ts +++ b/lib/datasource/rubygems/get-rubygems-org.ts @@ -1,6 +1,7 @@ import { logger } from '../../logger'; +import { ExternalHostError } from '../../types/error'; import { Http } from '../../util/http'; -import { DatasourceError, ReleaseResult } from '../common'; +import { ReleaseResult } from '../common'; import { id } from './common'; const http = new Http(id); @@ -38,7 +39,7 @@ async function updateRubyGemsVersions(): Promise { if (err.statusCode !== 416) { contentLength = 0; packageReleases = Object.create(null); // Because we might need a "constructor" key - throw new DatasourceError( + throw new ExternalHostError( new Error('Rubygems fetch error - need to reset cache') ); } diff --git a/lib/datasource/terraform-module/index.ts b/lib/datasource/terraform-module/index.ts index da0bce26adee16..4cac9a7d805a35 100644 --- a/lib/datasource/terraform-module/index.ts +++ b/lib/datasource/terraform-module/index.ts @@ -1,7 +1,8 @@ import { logger } from '../../logger'; +import { ExternalHostError } from '../../types/error'; import * as globalCache from '../../util/cache/global'; import { Http } from '../../util/http'; -import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common'; +import { GetReleasesConfig, ReleaseResult } from '../common'; export const id = 'terraform-module'; export const defaultRegistryUrls = ['https://registry.terraform.io']; @@ -113,7 +114,7 @@ export async function getReleases({ const failureCodes = ['EAI_AGAIN']; // istanbul ignore if if (failureCodes.includes(err.code)) { - throw new DatasourceError(err); + throw new ExternalHostError(err); } logger.warn( { err, lookupName }, diff --git a/lib/manager/gradle/index.ts b/lib/manager/gradle/index.ts index 0d1ea33e155350..2b1d02d87b1946 100644 --- a/lib/manager/gradle/index.ts +++ b/lib/manager/gradle/index.ts @@ -3,9 +3,9 @@ import * as os from 'os'; import * as fs from 'fs-extra'; import upath from 'upath'; import { LANGUAGE_JAVA } from '../../constants/languages'; -import { DatasourceError } from '../../datasource'; import * as datasourceMaven from '../../datasource/maven'; import { logger } from '../../logger'; +import { ExternalHostError } from '../../types/error'; import { ExecOptions, exec } from '../../util/exec'; import { BinarySource } from '../../util/exec/common'; import { readLocalFile } from '../../util/fs'; @@ -106,9 +106,7 @@ export async function executeGradle( ({ stdout, stderr } = await exec(cmd, execOptions)); } catch (err) /* istanbul ignore next */ { if (err.code === TIMEOUT_CODE) { - const error = new DatasourceError(err); - error.datasource = 'gradle'; - throw error; + throw new ExternalHostError(err, 'gradle'); } logger.warn({ errMessage: err.message }, 'Gradle extraction failed'); return; diff --git a/lib/manager/npm/post-update/index.ts b/lib/manager/npm/post-update/index.ts index 7ba4aae6f2832b..75cf85f9c3f6c7 100644 --- a/lib/manager/npm/post-update/index.ts +++ b/lib/manager/npm/post-update/index.ts @@ -3,9 +3,10 @@ import fs from 'fs-extra'; import upath from 'upath'; // eslint-disable-next-line import/no-unresolved import { SYSTEM_INSUFFICIENT_DISK_SPACE } from '../../../constants/error-messages'; -import { DatasourceError } from '../../../datasource/common'; +import { id as npmId } from '../../../datasource/npm'; import { logger } from '../../../logger'; import { platform } from '../../../platform'; +import { ExternalHostError } from '../../../types/error'; import { getChildProcessEnv } from '../../../util/exec/env'; import { deleteLocalFile } from '../../../util/fs'; import * as hostRules from '../../../util/host-rules'; @@ -432,7 +433,7 @@ export async function getAdditionalFiles( const err = new Error( 'lock file failed for the dependency being updated - skipping branch creation' ); - throw new DatasourceError(err); + throw new ExternalHostError(err, npmId); } } } @@ -492,10 +493,11 @@ export async function getAdditionalFiles( { dependency: upgrade.depName, type: 'yarn' }, 'lock file failed for the dependency being updated - skipping branch creation' ); - throw new DatasourceError( + throw new ExternalHostError( new Error( 'lock file failed for the dependency being updated - skipping branch creation' - ) + ), + npmId ); } /* eslint-enable no-useless-escape */ @@ -594,10 +596,11 @@ export async function getAdditionalFiles( { dependency: upgrade.depName, type: 'pnpm' }, 'lock file failed for the dependency being updated - skipping branch creation' ); - throw new DatasourceError( + throw new ExternalHostError( Error( 'lock file failed for the dependency being updated - skipping branch creation' - ) + ), + npmId ); } } @@ -672,10 +675,11 @@ export async function getAdditionalFiles( { dependency: upgrade.depName, type: 'yarn' }, 'lock file failed for the dependency being updated - skipping branch creation' ); - throw new DatasourceError( + throw new ExternalHostError( Error( 'lock file failed for the dependency being updated - skipping branch creation' - ) + ), + npmId ); } /* eslint-enable no-useless-escape */ @@ -688,10 +692,11 @@ export async function getAdditionalFiles( { dependency: upgrade.depName, type: 'npm' }, 'lock file failed for the dependency being updated - skipping branch creation' ); - throw new DatasourceError( + throw new ExternalHostError( Error( 'lock file failed for the dependency being updated - skipping branch creation' - ) + ), + npmId ); } } diff --git a/lib/manager/npm/post-update/yarn.ts b/lib/manager/npm/post-update/yarn.ts index 81d2d6e8d7e32a..1dcc798c0b1884 100644 --- a/lib/manager/npm/post-update/yarn.ts +++ b/lib/manager/npm/post-update/yarn.ts @@ -4,8 +4,9 @@ import { validRange } from 'semver'; import { quote } from 'shlex'; import { join } from 'upath'; import { SYSTEM_INSUFFICIENT_DISK_SPACE } from '../../../constants/error-messages'; -import { DatasourceError } from '../../../datasource'; +import { id as npmId } from '../../../datasource/npm'; import { logger } from '../../../logger'; +import { ExternalHostError } from '../../../types/error'; import { ExecOptions, exec } from '../../../util/exec'; import { PostUpdateConfig, Upgrade } from '../../common'; import { getNodeConstraint } from './node-version'; @@ -151,7 +152,7 @@ export async function generateLockFile( err.stderr.includes('getaddrinfo ENOTFOUND registry.yarnpkg.com') || err.stderr.includes('getaddrinfo ENOTFOUND registry.npmjs.org') ) { - throw new DatasourceError(err); + throw new ExternalHostError(err, npmId); } } return { error: true, stderr: err.stderr }; diff --git a/lib/platform/git/storage.ts b/lib/platform/git/storage.ts index 1784abbfc1ebc3..b857ed7f36e753 100644 --- a/lib/platform/git/storage.ts +++ b/lib/platform/git/storage.ts @@ -4,13 +4,13 @@ import fs from 'fs-extra'; import Git from 'simple-git/promise'; import { CONFIG_VALIDATION, - PLATFORM_FAILURE, REPOSITORY_CHANGED, REPOSITORY_EMPTY, REPOSITORY_TEMPORARY_ERROR, SYSTEM_INSUFFICIENT_DISK_SPACE, } from '../../constants/error-messages'; import { logger } from '../../logger'; +import { ExternalHostError } from '../../types/error'; import * as limits from '../../workers/global/limits'; import { CommitFilesConfig } from '../common'; import { writePrivateKey } from './private-key'; @@ -54,7 +54,7 @@ function checkForPlatformFailure(err: Error): void { ]; for (const errorStr of platformFailureStrings) { if (err.message.includes(errorStr)) { - throw new Error(PLATFORM_FAILURE); + throw new ExternalHostError(err, 'git'); } } } @@ -176,7 +176,7 @@ export class Storage { if (err.message?.includes('write error: No space left on device')) { throw new Error(SYSTEM_INSUFFICIENT_DISK_SPACE); } - throw new Error(PLATFORM_FAILURE); + throw new ExternalHostError(err, 'git'); } const durationMs = Math.round(Date.now() - cloneStart); logger.debug({ durationMs }, 'git clone completed'); diff --git a/lib/platform/github/index.ts b/lib/platform/github/index.ts index 62f77f00ab60f4..644643b558675e 100644 --- a/lib/platform/github/index.ts +++ b/lib/platform/github/index.ts @@ -3,7 +3,6 @@ import is from '@sindresorhus/is'; import delay from 'delay'; import { configFileNames } from '../../config/app-strings'; import { - PLATFORM_FAILURE, PLATFORM_INTEGRATION_UNAUTHORIZED, REPOSITORY_ACCESS_FORBIDDEN, REPOSITORY_ARCHIVED, @@ -24,6 +23,7 @@ import { } from '../../constants/pull-requests'; import { logger } from '../../logger'; import { BranchStatus } from '../../types'; +import { ExternalHostError } from '../../types/error'; import * as hostRules from '../../util/host-rules'; import * as githubHttp from '../../util/http/github'; import { sanitize } from '../../util/sanitize'; @@ -397,7 +397,7 @@ export async function initRepo({ } ); } catch (err) /* istanbul ignore next */ { - if (err.message === PLATFORM_FAILURE) { + if (err instanceof ExternalHostError) { throw err; } if ( @@ -965,7 +965,7 @@ export async function getPrList(): Promise { ); } catch (err) /* istanbul ignore next */ { logger.debug({ err }, 'getPrList err'); - throw new Error('platform-failure'); + throw new ExternalHostError(err, PLATFORM_TYPE_GITHUB); } config.prList = res.body.map((pr) => ({ number: pr.number, @@ -1094,7 +1094,7 @@ export async function getBranchStatus( logger.debug({ result: checkRunsRaw }, 'No check runs found'); } } catch (err) /* istanbul ignore next */ { - if (err.message === PLATFORM_FAILURE) { + if (err instanceof ExternalHostError) { throw err; } if ( @@ -1506,7 +1506,7 @@ async function getComments(issueNo: number): Promise { } catch (err) /* istanbul ignore next */ { if (err.statusCode === 404) { logger.debug('404 respose when retrieving comments'); - throw new Error(PLATFORM_FAILURE); + throw new ExternalHostError(err, PLATFORM_TYPE_GITHUB); } throw err; } @@ -1559,7 +1559,7 @@ export async function ensureComment({ } return true; } catch (err) /* istanbul ignore next */ { - if (err.message === PLATFORM_FAILURE) { + if (err instanceof ExternalHostError) { throw err; } if (err.body?.message?.includes('is locked')) { @@ -1690,7 +1690,7 @@ export async function updatePr( ); logger.debug({ pr: prNo }, 'PR updated'); } catch (err) /* istanbul ignore next */ { - if (err.message === PLATFORM_FAILURE) { + if (err instanceof ExternalHostError) { throw err; } logger.warn({ err }, 'Error updating PR'); diff --git a/lib/types/error.ts b/lib/types/error.ts new file mode 100644 index 00000000000000..b6e67592d0b28a --- /dev/null +++ b/lib/types/error.ts @@ -0,0 +1,19 @@ +import { EXTERNAL_HOST_ERROR } from '../constants/error-messages'; + +export class ExternalHostError extends Error { + hostType: string; + + err: Error; + + lookupName?: string; + + reason?: string; + + constructor(err: Error, hostType?: string) { + super(EXTERNAL_HOST_ERROR); + // Set the prototype explicitly: https://github.com/Microsoft/TypeScript/wiki/Breaking-Changes#extending-built-ins-like-error-array-and-map-may-no-longer-work + Object.setPrototypeOf(this, ExternalHostError.prototype); + this.hostType = hostType; + this.err = err; + } +} diff --git a/lib/util/http/github.spec.ts b/lib/util/http/github.spec.ts index 444d9ef06c0cbd..fb4e5b38aa35fc 100644 --- a/lib/util/http/github.spec.ts +++ b/lib/util/http/github.spec.ts @@ -2,8 +2,8 @@ import delay from 'delay'; import * as httpMock from '../../../test/httpMock'; import { getName } from '../../../test/util'; import { + EXTERNAL_HOST_ERROR, PLATFORM_BAD_CREDENTIALS, - PLATFORM_FAILURE, PLATFORM_INTEGRATION_UNAUTHORIZED, PLATFORM_RATE_LIMIT_EXCEEDED, REPOSITORY_CHANGED, @@ -126,7 +126,7 @@ describe(getName(__filename), () => { }, }); expect(e).toBeDefined(); - expect(e.message).toEqual(PLATFORM_FAILURE); + expect(e.message).toEqual(EXTERNAL_HOST_ERROR); }); it('should throw platform failure for ENOTFOUND, ETIMEDOUT or EAI_AGAIN', () => { const codes = ['ENOTFOUND', 'ETIMEDOUT', 'EAI_AGAIN']; @@ -137,7 +137,7 @@ describe(getName(__filename), () => { code, }); expect(e).toBeDefined(); - expect(e.message).toEqual(PLATFORM_FAILURE); + expect(e.message).toEqual(EXTERNAL_HOST_ERROR); } }); it('should throw platform failure for 500', () => { @@ -146,14 +146,14 @@ describe(getName(__filename), () => { message: 'Internal Server Error', }); expect(e).toBeDefined(); - expect(e.message).toEqual(PLATFORM_FAILURE); + expect(e.message).toEqual(EXTERNAL_HOST_ERROR); }); it('should throw platform failure ParseError', () => { const e = getError({ name: 'ParseError', }); expect(e).toBeDefined(); - expect(e.message).toEqual(PLATFORM_FAILURE); + expect(e.message).toEqual(EXTERNAL_HOST_ERROR); }); it('should throw for unauthorized integration', () => { const e = getError({ @@ -200,7 +200,7 @@ describe(getName(__filename), () => { }; const e = getError(gotErr); expect(e).toBeDefined(); - expect(e.message).toEqual(PLATFORM_FAILURE); + expect(e.message).toEqual(EXTERNAL_HOST_ERROR); }); it('should throw original error when failed to add reviewers', () => { const gotErr = { diff --git a/lib/util/http/github.ts b/lib/util/http/github.ts index 76c7d9d76db751..c2f6dd1b657c28 100644 --- a/lib/util/http/github.ts +++ b/lib/util/http/github.ts @@ -4,13 +4,13 @@ import pAll from 'p-all'; import parseLinkHeader from 'parse-link-header'; import { PLATFORM_BAD_CREDENTIALS, - PLATFORM_FAILURE, PLATFORM_INTEGRATION_UNAUTHORIZED, PLATFORM_RATE_LIMIT_EXCEEDED, REPOSITORY_CHANGED, } from '../../constants/error-messages'; import { PLATFORM_TYPE_GITHUB } from '../../constants/platforms'; import { logger } from '../../logger'; +import { ExternalHostError } from '../../types/error'; import { maskToken } from '../mask'; import { Http, HttpPostOptions, HttpResponse, InternalHttpOptions } from '.'; @@ -61,15 +61,15 @@ export function handleGotError( err.code === 'EAI_AGAIN') ) { logger.debug({ err }, 'GitHub failure: RequestError'); - throw new Error(PLATFORM_FAILURE); + throw new ExternalHostError(err, PLATFORM_TYPE_GITHUB); } if (err.name === 'ParseError') { logger.debug({ err }, ''); - throw new Error(PLATFORM_FAILURE); + throw new ExternalHostError(err, PLATFORM_TYPE_GITHUB); } if (err.statusCode >= 500 && err.statusCode < 600) { logger.debug({ err }, 'GitHub failure: 5xx'); - throw new Error(PLATFORM_FAILURE); + throw new ExternalHostError(err, PLATFORM_TYPE_GITHUB); } if ( err.statusCode === 403 && @@ -106,7 +106,7 @@ export function handleGotError( 'GitHub failure: Bad credentials' ); if (rateLimit === '60') { - throw new Error(PLATFORM_FAILURE); + throw new ExternalHostError(err, PLATFORM_TYPE_GITHUB); } throw new Error(PLATFORM_BAD_CREDENTIALS); } @@ -123,7 +123,7 @@ export function handleGotError( throw new Error(REPOSITORY_CHANGED); } logger.debug({ err }, '422 Error thrown from GitHub'); - throw new Error(PLATFORM_FAILURE); + throw new ExternalHostError(err, PLATFORM_TYPE_GITHUB); } if (err.statusCode === 404) { logger.debug({ url: err.url }, 'GitHub 404'); diff --git a/lib/util/http/gitlab.ts b/lib/util/http/gitlab.ts index 9a5d0f160509da..5e759d333952b2 100644 --- a/lib/util/http/gitlab.ts +++ b/lib/util/http/gitlab.ts @@ -1,8 +1,8 @@ import { URL } from 'url'; import parseLinkHeader from 'parse-link-header'; -import { PLATFORM_FAILURE } from '../../constants/error-messages'; import { PLATFORM_TYPE_GITLAB } from '../../constants/platforms'; import { logger } from '../../logger'; +import { ExternalHostError } from '../../types/error'; import { Http, HttpResponse, InternalHttpOptions } from '.'; let baseUrl = 'https://gitlab.com/api/v4/'; @@ -63,7 +63,7 @@ export class GitlabHttp extends Http { err.statusCode === 429 || (err.statusCode >= 500 && err.statusCode < 600) ) { - throw new Error(PLATFORM_FAILURE); + throw new ExternalHostError(err, PLATFORM_TYPE_GITLAB); } const platformFailureCodes = [ 'EAI_AGAIN', @@ -72,10 +72,10 @@ export class GitlabHttp extends Http { 'UNABLE_TO_VERIFY_LEAF_SIGNATURE', ]; if (platformFailureCodes.includes(err.code)) { - throw new Error(PLATFORM_FAILURE); + throw new ExternalHostError(err, PLATFORM_TYPE_GITLAB); } if (err.name === 'ParseError') { - throw new Error(PLATFORM_FAILURE); + throw new ExternalHostError(err, PLATFORM_TYPE_GITLAB); } throw err; } diff --git a/lib/workers/branch/index.ts b/lib/workers/branch/index.ts index 31963c57719a7c..a646d6dc708264 100644 --- a/lib/workers/branch/index.ts +++ b/lib/workers/branch/index.ts @@ -4,11 +4,9 @@ import { DateTime } from 'luxon'; import minimatch from 'minimatch'; import { RenovateConfig } from '../../config'; import { - DATASOURCE_FAILURE, MANAGER_LOCKFILE_ERROR, PLATFORM_AUTHENTICATION_ERROR, PLATFORM_BAD_CREDENTIALS, - PLATFORM_FAILURE, PLATFORM_INTEGRATION_UNAUTHORIZED, PLATFORM_RATE_LIMIT_EXCEEDED, REPOSITORY_CHANGED, @@ -24,6 +22,7 @@ import { logger } from '../../logger'; import { getAdditionalFiles } from '../../manager/npm/post-update'; import { platform } from '../../platform'; import { BranchStatus } from '../../types'; +import { ExternalHostError } from '../../types/error'; import { emojify } from '../../util/emoji'; import { exec } from '../../util/exec'; import { readLocalFile, writeLocalFile } from '../../util/fs'; @@ -541,10 +540,7 @@ export async function processBranch( err.message.includes('fatal: Authentication failed') ) { throw new Error(PLATFORM_AUTHENTICATION_ERROR); - } else if ( - err.message !== DATASOURCE_FAILURE && - err.message !== PLATFORM_FAILURE - ) { + } else if (!(err instanceof ExternalHostError)) { logger.error({ err }, `Error updating branch: ${err.message}`); } // Don't throw here - we don't want to stop the other renovations @@ -655,11 +651,8 @@ export async function processBranch( } } catch (err) /* istanbul ignore next */ { if ( - [ - PLATFORM_RATE_LIMIT_EXCEEDED, - PLATFORM_FAILURE, - REPOSITORY_CHANGED, - ].includes(err.message) + err instanceof ExternalHostError || + [PLATFORM_RATE_LIMIT_EXCEEDED, REPOSITORY_CHANGED].includes(err.message) ) { logger.debug('Passing PR error up'); throw err; diff --git a/lib/workers/pr/index.ts b/lib/workers/pr/index.ts index 9fc7e586e968ca..17c2bcdca22108 100644 --- a/lib/workers/pr/index.ts +++ b/lib/workers/pr/index.ts @@ -2,7 +2,6 @@ import sampleSize from 'lodash/sampleSize'; import uniq from 'lodash/uniq'; import { RenovateConfig } from '../../config/common'; import { - PLATFORM_FAILURE, PLATFORM_INTEGRATION_UNAUTHORIZED, PLATFORM_RATE_LIMIT_EXCEEDED, REPOSITORY_CHANGED, @@ -10,6 +9,7 @@ import { import { logger } from '../../logger'; import { PlatformPrOptions, Pr, platform } from '../../platform'; import { BranchStatus } from '../../types'; +import { ExternalHostError } from '../../types/error'; import { BranchConfig, PrResult } from '../common'; import { getPrBody } from './body'; import { ChangeLogError } from './changelog'; @@ -427,9 +427,9 @@ export async function ensurePr( } catch (err) { // istanbul ignore if if ( + err instanceof ExternalHostError || err.message === REPOSITORY_CHANGED || err.message === PLATFORM_RATE_LIMIT_EXCEEDED || - err.message === PLATFORM_FAILURE || err.message === PLATFORM_INTEGRATION_UNAUTHORIZED ) { logger.debug('Passing error up'); diff --git a/lib/workers/repository/error.spec.ts b/lib/workers/repository/error.spec.ts index f0b10e9548f98e..5fce970db9260a 100644 --- a/lib/workers/repository/error.spec.ts +++ b/lib/workers/repository/error.spec.ts @@ -1,12 +1,11 @@ import { RenovateConfig, getConfig } from '../../../test/util'; import { CONFIG_VALIDATION, - DATASOURCE_FAILURE, + EXTERNAL_HOST_ERROR, MANAGER_LOCKFILE_ERROR, MANAGER_NO_PACKAGE_FILES, PLATFORM_AUTHENTICATION_ERROR, PLATFORM_BAD_CREDENTIALS, - PLATFORM_FAILURE, PLATFORM_INTEGRATION_UNAUTHORIZED, PLATFORM_RATE_LIMIT_EXCEEDED, REPOSITORY_ACCESS_FORBIDDEN, @@ -27,7 +26,7 @@ import { SYSTEM_INSUFFICIENT_MEMORY, UNKNOWN_ERROR, } from '../../constants/error-messages'; -import { DatasourceError } from '../../datasource/common'; +import { ExternalHostError } from '../../types/error'; import handleError from './error'; jest.mock('./error-config'); @@ -48,7 +47,6 @@ describe('workers/repository/error', () => { REPOSITORY_FORKED, MANAGER_NO_PACKAGE_FILES, CONFIG_VALIDATION, - DATASOURCE_FAILURE, REPOSITORY_ARCHIVED, REPOSITORY_MIRRORED, REPOSITORY_RENAMED, @@ -60,7 +58,6 @@ describe('workers/repository/error', () => { MANAGER_LOCKFILE_ERROR, SYSTEM_INSUFFICIENT_DISK_SPACE, SYSTEM_INSUFFICIENT_MEMORY, - PLATFORM_FAILURE, REPOSITORY_NO_VULNERABILITY, REPOSITORY_CANNOT_FORK, PLATFORM_INTEGRATION_UNAUTHORIZED, @@ -73,23 +70,26 @@ describe('workers/repository/error', () => { expect(res).toEqual(err); }); }); - it(`handles DatasourceError`, async () => { - const res = await handleError(config, new DatasourceError(new Error())); - expect(res).toEqual(DATASOURCE_FAILURE); + it(`handles ExternalHostError`, async () => { + const res = await handleError( + config, + new ExternalHostError(new Error(), 'some-host-type') + ); + expect(res).toEqual(EXTERNAL_HOST_ERROR); }); it('rewrites git 5xx error', async () => { const gitError = new Error( "fatal: unable to access 'https://**redacted**@gitlab.com/learnox/learnox.git/': The requested URL returned error: 500\n" ); const res = await handleError(config, gitError); - expect(res).toEqual(PLATFORM_FAILURE); + expect(res).toEqual(EXTERNAL_HOST_ERROR); }); it('rewrites git remote error', async () => { const gitError = new Error( 'fatal: remote error: access denied or repository not exported: /b/nw/bd/27/47/159945428/108610112.git\n' ); const res = await handleError(config, gitError); - expect(res).toEqual(PLATFORM_FAILURE); + expect(res).toEqual(EXTERNAL_HOST_ERROR); }); it('handles unknown error', async () => { const res = await handleError(config, new Error('abcdefg')); diff --git a/lib/workers/repository/error.ts b/lib/workers/repository/error.ts index 48f8e7a911a9af..5ff14d0aeac259 100644 --- a/lib/workers/repository/error.ts +++ b/lib/workers/repository/error.ts @@ -2,12 +2,11 @@ import { RenovateConfig } from '../../config'; import { CONFIG_VALIDATION, - DATASOURCE_FAILURE, + EXTERNAL_HOST_ERROR, MANAGER_LOCKFILE_ERROR, MANAGER_NO_PACKAGE_FILES, PLATFORM_AUTHENTICATION_ERROR, PLATFORM_BAD_CREDENTIALS, - PLATFORM_FAILURE, PLATFORM_INTEGRATION_UNAUTHORIZED, PLATFORM_RATE_LIMIT_EXCEEDED, REPOSITORY_ACCESS_FORBIDDEN, @@ -28,8 +27,8 @@ import { SYSTEM_INSUFFICIENT_MEMORY, UNKNOWN_ERROR, } from '../../constants/error-messages'; -import { DatasourceError } from '../../datasource/common'; import { logger } from '../../logger'; +import { ExternalHostError } from '../../types/error'; import { raiseConfigWarningIssue } from './error-config'; export default async function handleError( @@ -107,22 +106,12 @@ export default async function handleError( await raiseConfigWarningIssue(config, err); return err.message; } - if (err instanceof DatasourceError) { + if (err instanceof ExternalHostError) { logger.warn( - { datasource: err.datasource, lookupName: err.lookupName, err: err.err }, - 'Datasource failure' + { hostType: err.hostType, lookupName: err.lookupName, err: err.err }, + 'Host error' ); - logger.info('Registry error - skipping'); - delete config.branchList; // eslint-disable-line no-param-reassign - return err.message; - } - if (err.message === DATASOURCE_FAILURE) { - logger.info({ err }, 'Registry error - skipping'); - delete config.branchList; // eslint-disable-line no-param-reassign - return err.message; - } - if (err.message === PLATFORM_FAILURE) { - logger.info('Platform error - skipping'); + logger.info('External host error causing abort - skipping'); delete config.branchList; // eslint-disable-line no-param-reassign return err.message; } @@ -174,7 +163,7 @@ export default async function handleError( logger.warn({ err }, 'Git error - aborting'); delete config.branchList; // eslint-disable-line no-param-reassign // rewrite this error - return PLATFORM_FAILURE; + return EXTERNAL_HOST_ERROR; } if ( err.message.includes('The remote end hung up unexpectedly') || @@ -183,7 +172,7 @@ export default async function handleError( logger.warn({ err }, 'Git error - aborting'); delete config.branchList; // eslint-disable-line no-param-reassign // rewrite this error - return PLATFORM_FAILURE; + return EXTERNAL_HOST_ERROR; } // Swallow this error so that other repositories can be processed logger.error({ err }, `Repository has unknown error`); diff --git a/lib/workers/repository/init/config.ts b/lib/workers/repository/init/config.ts index 547c2b39335b7b..21ce3d73254720 100644 --- a/lib/workers/repository/init/config.ts +++ b/lib/workers/repository/init/config.ts @@ -7,13 +7,11 @@ import { configFileNames } from '../../../config/app-strings'; import { decryptConfig } from '../../../config/decrypt'; import { migrateAndValidate } from '../../../config/migrate-validate'; import * as presets from '../../../config/presets'; -import { - CONFIG_VALIDATION, - PLATFORM_FAILURE, -} from '../../../constants/error-messages'; +import { CONFIG_VALIDATION } from '../../../constants/error-messages'; import * as npmApi from '../../../datasource/npm'; import { logger } from '../../../logger'; import { platform } from '../../../platform'; +import { ExternalHostError } from '../../../types/error'; import { readLocalFile } from '../../../util/fs'; import * as hostRules from '../../../util/host-rules'; import { flattenPackageRules } from './flatten'; @@ -59,7 +57,10 @@ export async function mergeRenovateConfig( // istanbul ignore if if (renovateConfig === null) { logger.warn('Fetching renovate config returns null'); - throw new Error(PLATFORM_FAILURE); + throw new ExternalHostError( + Error('Fetching renovate config returns null'), + config.platform + ); } // istanbul ignore if if (!renovateConfig.length) { From aa9a00416d5fa83620eccf7d3caaf41f7c6e2ad3 Mon Sep 17 00:00:00 2001 From: Yusuke Yamada Date: Tue, 23 Jun 2020 04:29:27 +0900 Subject: [PATCH 05/25] feat(sbt): Support private variable version resolving (#6565) --- .../private-variable-dependency-file.scala | 15 +++++++++++ .../sbt/__snapshots__/extract.spec.ts.snap | 25 +++++++++++++++++++ lib/manager/sbt/extract.spec.ts | 10 ++++++++ lib/manager/sbt/extract.ts | 16 ++++++++---- 4 files changed, 61 insertions(+), 5 deletions(-) create mode 100644 lib/manager/sbt/__fixtures__/private-variable-dependency-file.scala diff --git a/lib/manager/sbt/__fixtures__/private-variable-dependency-file.scala b/lib/manager/sbt/__fixtures__/private-variable-dependency-file.scala new file mode 100644 index 00000000000000..6d61f40ad61827 --- /dev/null +++ b/lib/manager/sbt/__fixtures__/private-variable-dependency-file.scala @@ -0,0 +1,15 @@ +import sbt._ + +object Dependencies { + val moreSettings = Seq( + scalaVersion := "2.13.0-RC5" + ) + + private val abcVersion = "1.2.3" + + private lazy val ujson = "com.example" %% "foo" % "0.7.1" + + lazy val abc = "com.abc" % "abc" % abcVersion + + lazy val dependentLibraries = Seq(ujson, abc) +} diff --git a/lib/manager/sbt/__snapshots__/extract.spec.ts.snap b/lib/manager/sbt/__snapshots__/extract.spec.ts.snap index 8dc5a48769a641..436072c3216a52 100644 --- a/lib/manager/sbt/__snapshots__/extract.spec.ts.snap +++ b/lib/manager/sbt/__snapshots__/extract.spec.ts.snap @@ -1,5 +1,30 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`lib/manager/sbt/extract extractPackageFile() extract deps from native scala file with private variables 1`] = ` +Object { + "deps": Array [ + Object { + "currentValue": "0.7.1", + "datasource": "sbt-package", + "depName": "com.example:foo", + "lookupName": "com.example:foo_2.13.0-RC5", + "registryUrls": Array [ + "https://repo.maven.apache.org/maven2", + ], + }, + Object { + "currentValue": "1.2.3", + "datasource": "sbt-package", + "depName": "com.abc:abc", + "lookupName": "com.abc:abc", + "registryUrls": Array [ + "https://repo.maven.apache.org/maven2", + ], + }, + ], +} +`; + exports[`lib/manager/sbt/extract extractPackageFile() extract deps from native scala file with variables 1`] = ` Object { "deps": Array [ diff --git a/lib/manager/sbt/extract.spec.ts b/lib/manager/sbt/extract.spec.ts index fde42188d274e2..b2d8b2666be33e 100644 --- a/lib/manager/sbt/extract.spec.ts +++ b/lib/manager/sbt/extract.spec.ts @@ -20,6 +20,11 @@ const sbtDependencyFile = readFileSync( 'utf8' ); +const sbtPrivateVariableDependencyFile = readFileSync( + resolve(__dirname, `./__fixtures__/private-variable-dependency-file.scala`), + 'utf8' +); + describe('lib/manager/sbt/extract', () => { describe('extractPackageFile()', () => { it('returns null for empty', () => { @@ -78,5 +83,10 @@ describe('lib/manager/sbt/extract', () => { `; expect(extractPackageFile(content)).toMatchSnapshot(); }); + it('extract deps from native scala file with private variables', () => { + expect( + extractPackageFile(sbtPrivateVariableDependencyFile) + ).toMatchSnapshot(); + }); }); }); diff --git a/lib/manager/sbt/extract.ts b/lib/manager/sbt/extract.ts index c1336560b8ccf3..d6901ec38051d6 100644 --- a/lib/manager/sbt/extract.ts +++ b/lib/manager/sbt/extract.ts @@ -66,20 +66,26 @@ const getResolverUrl = (str: string): string => .replace(/"[\s,)]*$/, ''); const isVarDependency = (str: string): boolean => - /^\s*(lazy\s*)?val\s[_a-zA-Z][_a-zA-Z0-9]*\s*=.*(%%?).*%.*/.test(str); + /^\s*(private\s*)?(lazy\s*)?val\s[_a-zA-Z][_a-zA-Z0-9]*\s*=.*(%%?).*%.*/.test( + str + ); const isVarDef = (str: string): boolean => - /^\s*(lazy\s*)?val\s+[_a-zA-Z][_a-zA-Z0-9]*\s*=\s*"[^"]*"\s*$/.test(str); + /^\s*(private\s*)?(lazy\s*)?val\s+[_a-zA-Z][_a-zA-Z0-9]*\s*=\s*"[^"]*"\s*$/.test( + str + ); const getVarName = (str: string): string => - str.replace(/^\s*(lazy\s*)?val\s+/, '').replace(/\s*=\s*"[^"]*"\s*$/, ''); + str + .replace(/^\s*(private\s*)?(lazy\s*)?val\s+/, '') + .replace(/\s*=\s*"[^"]*"\s*$/, ''); const isVarName = (str: string): boolean => /^[_a-zA-Z][_a-zA-Z0-9]*$/.test(str); const getVarInfo = (str: string, ctx: ParseContext): { val: string } => { const rightPart = str.replace( - /^\s*(lazy\s*)?val\s+[_a-zA-Z][_a-zA-Z0-9]*\s*=\s*"/, + /^\s*(private\s*)?(lazy\s*)?val\s+[_a-zA-Z][_a-zA-Z0-9]*\s*=\s*"/, '' ); const val = rightPart.replace(/"\s*$/, ''); @@ -210,7 +216,7 @@ function parseSbtLine( } else if (isVarDependency(line)) { isMultiDeps = false; const depExpr = line.replace( - /^\s*(lazy\s*)?val\s[_a-zA-Z][_a-zA-Z0-9]*\s*=\s*/, + /^\s*(private\s*)?(lazy\s*)?val\s[_a-zA-Z][_a-zA-Z0-9]*\s*=\s*/, '' ); dep = parseDepExpr(depExpr, { From 02e6817fb977ba82c13ce076759dbe179c37265d Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Mon, 22 Jun 2020 21:57:17 +0200 Subject: [PATCH 06/25] test: move e2e tests (#6567) --- .gitignore | 2 +- package.json | 2 +- {e2e => test/e2e}/package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename {e2e => test/e2e}/package.json (85%) diff --git a/.gitignore b/.gitignore index ccfa943d5bbd66..c3fefd935eabe1 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,7 @@ /package-lock.json *.pyc renovate-0.0.0-semantic-release.tgz -/e2e/node_modules +/test/e2e/node_modules .eslintcache junit.xml /test-results diff --git a/package.json b/package.json index 26200f6d77e103..c472a3b2bf2a80 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "release": "node --experimental-modules tools/release.mjs", "start": "babel-node --extensions \".ts,.js\" -- lib/renovate.ts", "test-dirty": "git diff --exit-code", - "test-e2e": "npm pack && cd e2e && yarn install --no-lockfile --ignore-optional --prod && yarn test", + "test-e2e": "npm pack && cd test/e2e && yarn install --no-lockfile --ignore-optional --prod && yarn test", "test-schema": "babel-node --extensions \".ts,.js\" -- test/json-schema.ts", "test": "run-s lint test-schema type-check jest", "tsc": "tsc", diff --git a/e2e/package.json b/test/e2e/package.json similarity index 85% rename from e2e/package.json rename to test/e2e/package.json index 7774061760230b..9051c9b567e9cb 100644 --- a/e2e/package.json +++ b/test/e2e/package.json @@ -6,7 +6,7 @@ "test": "renovate --version && renovate-config-validator" }, "dependencies": { - "renovate": "../renovate-0.0.0-semantic-release.tgz" + "renovate": "../../renovate-0.0.0-semantic-release.tgz" }, "renovate-config": { "default": { From 4610e25b05c6c94bcebdad461c963b8d89c40969 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Mon, 22 Jun 2020 22:51:03 +0200 Subject: [PATCH 07/25] fix(clojure): set registryStrategy --- lib/datasource/clojure/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/datasource/clojure/index.ts b/lib/datasource/clojure/index.ts index 05dfdf989d0227..8e86fdded4ee9e 100644 --- a/lib/datasource/clojure/index.ts +++ b/lib/datasource/clojure/index.ts @@ -3,5 +3,6 @@ import { MAVEN_REPO } from '../maven/common'; export const id = 'clojure'; export const defaultRegistryUrls = ['https://clojars.org/repo', MAVEN_REPO]; +export const registryStrategy = 'merge'; export { getReleases } from '../maven'; From 9219709544cb0c8886bfdef60f2373192d9655d1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 22 Jun 2020 23:00:54 +0000 Subject: [PATCH 08/25] chore(deps): update babel monorepo to v7.10.3 (#6569) Co-authored-by: Renovate Bot --- package.json | 10 +- yarn.lock | 360 +++++++++++++++++++++++++-------------------------- 2 files changed, 185 insertions(+), 185 deletions(-) diff --git a/package.json b/package.json index c472a3b2bf2a80..a762b82457e84f 100644 --- a/package.json +++ b/package.json @@ -174,13 +174,13 @@ }, "devDependencies": { "@actions/core": "1.2.4", - "@babel/cli": "7.10.1", - "@babel/core": "7.10.2", - "@babel/node": "7.10.1", + "@babel/cli": "7.10.3", + "@babel/core": "7.10.3", + "@babel/node": "7.10.3", "@babel/plugin-proposal-class-properties": "7.10.1", - "@babel/plugin-proposal-object-rest-spread": "7.10.1", + "@babel/plugin-proposal-object-rest-spread": "7.10.3", "@babel/plugin-syntax-dynamic-import": "7.8.3", - "@babel/preset-env": "7.10.2", + "@babel/preset-env": "7.10.3", "@babel/preset-typescript": "7.10.1", "@jest/globals": "26.0.1", "@jest/reporters": "26.0.1", diff --git a/yarn.lock b/yarn.lock index 601f72e2f62982..3c3147f451a97e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7,10 +7,10 @@ resolved "https://registry.yarnpkg.com/@actions/core/-/core-1.2.4.tgz#96179dbf9f8d951dd74b40a0dbd5c22555d186ab" integrity sha512-YJCEq8BE3CdN8+7HPZ/4DxJjk/OkZV2FFIf+DlZTC/4iBlzYCD5yjRR6eiOS5llO11zbRltIRuKAjMKaWTE6cg== -"@babel/cli@7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/cli/-/cli-7.10.1.tgz#b6e5cd43a17b8f639442ab027976408ebe6d79a0" - integrity sha512-cVB+dXeGhMOqViIaZs3A9OUAe4pKw4SBNdMw6yHJMYR7s4TB+Cei7ThquV/84O19PdIFWuwe03vxxES0BHUm5g== +"@babel/cli@7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/cli/-/cli-7.10.3.tgz#4ea83bd997d2a41c78d07263ada3ec466fb3764b" + integrity sha512-lWB3yH5/fWY8pi2Kj5/fA+17guJ9feSBw5DNjTju3/nRi9sXnl1JPh7aKQOSvdNbiDbkzzoGYtsr46M8dGmXDQ== dependencies: commander "^4.0.1" convert-source-map "^1.1.0" @@ -23,35 +23,35 @@ optionalDependencies: chokidar "^2.1.8" -"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.10.1.tgz#d5481c5095daa1c57e16e54c6f9198443afb49ff" - integrity sha512-IGhtTmpjGbYzcEDOw7DcQtbQSXcG9ftmAXtWTu9V936vDye4xjjekktFAtgZsWpzTj/X01jocB46mTywm/4SZw== +"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.10.3.tgz#324bcfd8d35cd3d47dae18cde63d752086435e9a" + integrity sha512-fDx9eNW0qz0WkUeqL6tXEXzVlPh6Y5aCDEZesl0xBGA8ndRukX91Uk44ZqnkECp01NAZUdCAl+aiQNGi0k88Eg== dependencies: - "@babel/highlight" "^7.10.1" + "@babel/highlight" "^7.10.3" -"@babel/compat-data@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.10.1.tgz#b1085ffe72cd17bf2c0ee790fc09f9626011b2db" - integrity sha512-CHvCj7So7iCkGKPRFUfryXIkU2gSBw7VSZFYLsqVhrS47269VK2Hfi9S/YcublPMW8k1u2bQBlbDruoQEm4fgw== +"@babel/compat-data@^7.10.1", "@babel/compat-data@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.10.3.tgz#9af3e033f36e8e2d6e47570db91e64a846f5d382" + integrity sha512-BDIfJ9uNZuI0LajPfoYV28lX8kyCPMHY6uY4WH1lJdcicmAfxCK5ASzaeV0D/wsUaRH/cLk+amuxtC37sZ8TUg== dependencies: browserslist "^4.12.0" invariant "^2.2.4" semver "^5.5.0" -"@babel/core@7.10.2", "@babel/core@^7.1.0", "@babel/core@^7.7.5": - version "7.10.2" - resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.10.2.tgz#bd6786046668a925ac2bd2fd95b579b92a23b36a" - integrity sha512-KQmV9yguEjQsXqyOUGKjS4+3K8/DlOCE2pZcq4augdQmtTy5iv5EHtmMSJ7V4c1BIPjuwtZYqYLCq9Ga+hGBRQ== +"@babel/core@7.10.3", "@babel/core@^7.1.0", "@babel/core@^7.7.5": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.10.3.tgz#73b0e8ddeec1e3fdd7a2de587a60e17c440ec77e" + integrity sha512-5YqWxYE3pyhIi84L84YcwjeEgS+fa7ZjK6IBVGTjDVfm64njkR2lfDhVR5OudLk8x2GK59YoSyVv+L/03k1q9w== dependencies: - "@babel/code-frame" "^7.10.1" - "@babel/generator" "^7.10.2" + "@babel/code-frame" "^7.10.3" + "@babel/generator" "^7.10.3" "@babel/helper-module-transforms" "^7.10.1" "@babel/helpers" "^7.10.1" - "@babel/parser" "^7.10.2" - "@babel/template" "^7.10.1" - "@babel/traverse" "^7.10.1" - "@babel/types" "^7.10.2" + "@babel/parser" "^7.10.3" + "@babel/template" "^7.10.3" + "@babel/traverse" "^7.10.3" + "@babel/types" "^7.10.3" convert-source-map "^1.7.0" debug "^4.1.0" gensync "^1.0.0-beta.1" @@ -61,12 +61,12 @@ semver "^5.4.1" source-map "^0.5.0" -"@babel/generator@^7.10.1", "@babel/generator@^7.10.2": - version "7.10.2" - resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.10.2.tgz#0fa5b5b2389db8bfdfcc3492b551ee20f5dd69a9" - integrity sha512-AxfBNHNu99DTMvlUPlt1h2+Hn7knPpH5ayJ8OqDWSeLld+Fi2AYBTC/IejWDM9Edcii4UzZRCsbUt0WlSDsDsA== +"@babel/generator@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.10.3.tgz#32b9a0d963a71d7a54f5f6c15659c3dbc2a523a5" + integrity sha512-drt8MUHbEqRzNR0xnF8nMehbY11b1SDkRw03PSNH/3Rb2Z35oxkddVSi3rcaak0YJQ86PCuE7Qx1jSFhbLNBMA== dependencies: - "@babel/types" "^7.10.2" + "@babel/types" "^7.10.3" jsesc "^2.5.1" lodash "^4.17.13" source-map "^0.5.0" @@ -118,13 +118,13 @@ "@babel/helper-regex" "^7.10.1" regexpu-core "^4.7.0" -"@babel/helper-define-map@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/helper-define-map/-/helper-define-map-7.10.1.tgz#5e69ee8308648470dd7900d159c044c10285221d" - integrity sha512-+5odWpX+OnvkD0Zmq7panrMuAGQBu6aPUgvMzuMGo4R+jUOvealEj2hiqI6WhxgKrTpFoFj0+VdsuA8KDxHBDg== +"@babel/helper-define-map@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/helper-define-map/-/helper-define-map-7.10.3.tgz#d27120a5e57c84727b30944549b2dfeca62401a8" + integrity sha512-bxRzDi4Sin/k0drWCczppOhov1sBSdBvXJObM1NLHQzjhXhwRtn7aRWGvLJWCYbuu2qUk3EKs6Ci9C9ps8XokQ== dependencies: - "@babel/helper-function-name" "^7.10.1" - "@babel/types" "^7.10.1" + "@babel/helper-function-name" "^7.10.3" + "@babel/types" "^7.10.3" lodash "^4.17.13" "@babel/helper-explode-assignable-expression@^7.10.1": @@ -135,28 +135,28 @@ "@babel/traverse" "^7.10.1" "@babel/types" "^7.10.1" -"@babel/helper-function-name@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.10.1.tgz#92bd63829bfc9215aca9d9defa85f56b539454f4" - integrity sha512-fcpumwhs3YyZ/ttd5Rz0xn0TpIwVkN7X0V38B9TWNfVF42KEkhkAAuPCQ3oXmtTRtiPJrmZ0TrfS0GKF0eMaRQ== +"@babel/helper-function-name@^7.10.1", "@babel/helper-function-name@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.10.3.tgz#79316cd75a9fa25ba9787ff54544307ed444f197" + integrity sha512-FvSj2aiOd8zbeqijjgqdMDSyxsGHaMt5Tr0XjQsGKHD3/1FP3wksjnLAWzxw7lvXiej8W1Jt47SKTZ6upQNiRw== dependencies: - "@babel/helper-get-function-arity" "^7.10.1" - "@babel/template" "^7.10.1" - "@babel/types" "^7.10.1" + "@babel/helper-get-function-arity" "^7.10.3" + "@babel/template" "^7.10.3" + "@babel/types" "^7.10.3" -"@babel/helper-get-function-arity@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.10.1.tgz#7303390a81ba7cb59613895a192b93850e373f7d" - integrity sha512-F5qdXkYGOQUb0hpRaPoetF9AnsXknKjWMZ+wmsIRsp5ge5sFh4c3h1eH2pRTTuy9KKAA2+TTYomGXAtEL2fQEw== +"@babel/helper-get-function-arity@^7.10.1", "@babel/helper-get-function-arity@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.10.3.tgz#3a28f7b28ccc7719eacd9223b659fdf162e4c45e" + integrity sha512-iUD/gFsR+M6uiy69JA6fzM5seno8oE85IYZdbVVEuQaZlEzMO2MXblh+KSPJgsZAUx0EEbWXU0yJaW7C9CdAVg== dependencies: - "@babel/types" "^7.10.1" + "@babel/types" "^7.10.3" -"@babel/helper-hoist-variables@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.10.1.tgz#7e77c82e5dcae1ebf123174c385aaadbf787d077" - integrity sha512-vLm5srkU8rI6X3+aQ1rQJyfjvCBLXP8cAGeuw04zeAM2ItKb1e7pmVmLyHb4sDaAYnLL13RHOZPLEtcGZ5xvjg== +"@babel/helper-hoist-variables@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.10.3.tgz#d554f52baf1657ffbd7e5137311abc993bb3f068" + integrity sha512-9JyafKoBt5h20Yv1+BXQMdcXXavozI1vt401KBiRc2qzUepbVnd7ogVNymY1xkQN9fekGwfxtotH2Yf5xsGzgg== dependencies: - "@babel/types" "^7.10.1" + "@babel/types" "^7.10.3" "@babel/helper-member-expression-to-functions@^7.10.1": version "7.10.1" @@ -165,12 +165,12 @@ dependencies: "@babel/types" "^7.10.1" -"@babel/helper-module-imports@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.10.1.tgz#dd331bd45bccc566ce77004e9d05fe17add13876" - integrity sha512-SFxgwYmZ3HZPyZwJRiVNLRHWuW2OgE5k2nrVs6D9Iv4PPnXVffuEHy83Sfx/l4SqF+5kyJXjAyUmrG7tNm+qVg== +"@babel/helper-module-imports@^7.10.1", "@babel/helper-module-imports@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.10.3.tgz#766fa1d57608e53e5676f23ae498ec7a95e1b11a" + integrity sha512-Jtqw5M9pahLSUWA+76nhK9OG8nwYXzhQzVIGFoNaHnXF/r4l7kz4Fl0UAW7B6mqC5myoJiBP5/YQlXQTMfHI9w== dependencies: - "@babel/types" "^7.10.1" + "@babel/types" "^7.10.3" "@babel/helper-module-transforms@^7.10.1": version "7.10.1" @@ -185,17 +185,17 @@ "@babel/types" "^7.10.1" lodash "^4.17.13" -"@babel/helper-optimise-call-expression@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.10.1.tgz#b4a1f2561870ce1247ceddb02a3860fa96d72543" - integrity sha512-a0DjNS1prnBsoKx83dP2falChcs7p3i8VMzdrSbfLhuQra/2ENC4sbri34dz/rWmDADsmF1q5GbfaXydh0Jbjg== +"@babel/helper-optimise-call-expression@^7.10.1", "@babel/helper-optimise-call-expression@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.10.3.tgz#f53c4b6783093195b0f69330439908841660c530" + integrity sha512-kT2R3VBH/cnSz+yChKpaKRJQJWxdGoc6SjioRId2wkeV3bK0wLLioFpJROrX0U4xr/NmxSSAWT/9Ih5snwIIzg== dependencies: - "@babel/types" "^7.10.1" + "@babel/types" "^7.10.3" -"@babel/helper-plugin-utils@^7.0.0", "@babel/helper-plugin-utils@^7.10.1", "@babel/helper-plugin-utils@^7.8.0": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.1.tgz#ec5a5cf0eec925b66c60580328b122c01230a127" - integrity sha512-fvoGeXt0bJc7VMWZGCAEBEMo/HAjW2mP8apF5eXK0wSqwLAVHAISCWRoLMBMUs2kqeaG77jltVqu4Hn8Egl3nA== +"@babel/helper-plugin-utils@^7.0.0", "@babel/helper-plugin-utils@^7.10.1", "@babel/helper-plugin-utils@^7.10.3", "@babel/helper-plugin-utils@^7.8.0": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.3.tgz#aac45cccf8bc1873b99a85f34bceef3beb5d3244" + integrity sha512-j/+j8NAWUTxOtx4LKHybpSClxHoq6I91DQ/mKgAXn5oNUPIUiGppjPIX3TDtJWPrdfP9Kfl7e4fgVMiQR9VE/g== "@babel/helper-regex@^7.10.1": version "7.10.1" @@ -204,16 +204,16 @@ dependencies: lodash "^4.17.13" -"@babel/helper-remap-async-to-generator@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.10.1.tgz#bad6aaa4ff39ce8d4b82ccaae0bfe0f7dbb5f432" - integrity sha512-RfX1P8HqsfgmJ6CwaXGKMAqbYdlleqglvVtht0HGPMSsy2V6MqLlOJVF/0Qyb/m2ZCi2z3q3+s6Pv7R/dQuZ6A== +"@babel/helper-remap-async-to-generator@^7.10.1", "@babel/helper-remap-async-to-generator@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.10.3.tgz#18564f8a6748be466970195b876e8bba3bccf442" + integrity sha512-sLB7666ARbJUGDO60ZormmhQOyqMX/shKBXZ7fy937s+3ID8gSrneMvKSSb+8xIM5V7Vn6uNVtOY1vIm26XLtA== dependencies: "@babel/helper-annotate-as-pure" "^7.10.1" "@babel/helper-wrap-function" "^7.10.1" - "@babel/template" "^7.10.1" - "@babel/traverse" "^7.10.1" - "@babel/types" "^7.10.1" + "@babel/template" "^7.10.3" + "@babel/traverse" "^7.10.3" + "@babel/types" "^7.10.3" "@babel/helper-replace-supers@^7.10.1": version "7.10.1" @@ -240,10 +240,10 @@ dependencies: "@babel/types" "^7.10.1" -"@babel/helper-validator-identifier@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.10.1.tgz#5770b0c1a826c4f53f5ede5e153163e0318e94b5" - integrity sha512-5vW/JXLALhczRCWP0PnFDMCJAchlBvM7f4uk/jXritBnIa6E1KmqmtrS3yn1LAnxFBypQ3eneLuXjsnfQsgILw== +"@babel/helper-validator-identifier@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.10.3.tgz#60d9847f98c4cea1b279e005fdb7c28be5412d15" + integrity sha512-bU8JvtlYpJSBPuj1VUmKpFGaDZuLxASky3LhaKj3bmpSTY6VWooSM8msk+Z0CZoErFye2tlABF6yDkT3FOPAXw== "@babel/helper-wrap-function@^7.10.1": version "7.10.1" @@ -264,21 +264,21 @@ "@babel/traverse" "^7.10.1" "@babel/types" "^7.10.1" -"@babel/highlight@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.10.1.tgz#841d098ba613ba1a427a2b383d79e35552c38ae0" - integrity sha512-8rMof+gVP8mxYZApLF/JgNDAkdKa+aJt3ZYxF8z6+j/hpeXL7iMsKCPHa2jNMHu/qqBwzQF4OHNoYi8dMA/rYg== +"@babel/highlight@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.10.3.tgz#c633bb34adf07c5c13156692f5922c81ec53f28d" + integrity sha512-Ih9B/u7AtgEnySE2L2F0Xm0GaM729XqqLfHkalTsbjXGyqmf/6M0Cu0WpvqueUlW+xk88BHw9Nkpj49naU+vWw== dependencies: - "@babel/helper-validator-identifier" "^7.10.1" + "@babel/helper-validator-identifier" "^7.10.3" chalk "^2.0.0" js-tokens "^4.0.0" -"@babel/node@7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/node/-/node-7.10.1.tgz#34d3405c7c3d5139c48cbce2c80226f46e36ab25" - integrity sha512-HoLxelFIekiipykhN0d3cTSLZVxnl0aZiwv6oW4mxjeQEMOt1J/YGnBaIDyYWQ5tIHkUL1cqqn8LOvmWhFoCyw== +"@babel/node@7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/node/-/node-7.10.3.tgz#38fc470452e7e60f624cb5cf1134436141a2d5f5" + integrity sha512-HCOYT30KK3gHlzGon4ANQdqjiOFv50SkYOqxdUp8DiO/fxMNlRBR6mjdQy2SgvoYTrodGeiwJM1e43ehcMiGUQ== dependencies: - "@babel/register" "^7.10.1" + "@babel/register" "^7.10.3" commander "^4.0.1" core-js "^3.2.1" lodash "^4.17.13" @@ -287,18 +287,18 @@ resolve "^1.13.1" v8flags "^3.1.1" -"@babel/parser@^7.1.0", "@babel/parser@^7.10.1", "@babel/parser@^7.10.2": - version "7.10.2" - resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.10.2.tgz#871807f10442b92ff97e4783b9b54f6a0ca812d0" - integrity sha512-PApSXlNMJyB4JiGVhCOlzKIif+TKFTvu0aQAhnTvfP/z3vVSN6ZypH5bfUNwFXXjRQtUEBNFd2PtmCmG2Py3qQ== +"@babel/parser@^7.1.0", "@babel/parser@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.10.3.tgz#7e71d892b0d6e7d04a1af4c3c79d72c1f10f5315" + integrity sha512-oJtNJCMFdIMwXGmx+KxuaD7i3b8uS7TTFYW/FNG2BT8m+fmGHoiPYoH0Pe3gya07WuFmM5FCDIr1x0irkD/hyA== -"@babel/plugin-proposal-async-generator-functions@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.10.1.tgz#6911af5ba2e615c4ff3c497fe2f47b35bf6d7e55" - integrity sha512-vzZE12ZTdB336POZjmpblWfNNRpMSua45EYnRigE2XsZxcXcIyly2ixnTJasJE4Zq3U7t2d8rRF7XRUuzHxbOw== +"@babel/plugin-proposal-async-generator-functions@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.10.3.tgz#5a02453d46e5362e2073c7278beab2e53ad7d939" + integrity sha512-WUUWM7YTOudF4jZBAJIW9D7aViYC/Fn0Pln4RIHlQALyno3sXSjqmTA4Zy1TKC2D49RCR8Y/Pn4OIUtEypK3CA== dependencies: - "@babel/helper-plugin-utils" "^7.10.1" - "@babel/helper-remap-async-to-generator" "^7.10.1" + "@babel/helper-plugin-utils" "^7.10.3" + "@babel/helper-remap-async-to-generator" "^7.10.3" "@babel/plugin-syntax-async-generators" "^7.8.0" "@babel/plugin-proposal-class-properties@7.10.1", "@babel/plugin-proposal-class-properties@^7.10.1": @@ -341,12 +341,12 @@ "@babel/helper-plugin-utils" "^7.10.1" "@babel/plugin-syntax-numeric-separator" "^7.10.1" -"@babel/plugin-proposal-object-rest-spread@7.10.1", "@babel/plugin-proposal-object-rest-spread@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.10.1.tgz#cba44908ac9f142650b4a65b8aa06bf3478d5fb6" - integrity sha512-Z+Qri55KiQkHh7Fc4BW6o+QBuTagbOp9txE+4U1i79u9oWlf2npkiDx+Rf3iK3lbcHBuNy9UOkwuR5wOMH3LIQ== +"@babel/plugin-proposal-object-rest-spread@7.10.3", "@babel/plugin-proposal-object-rest-spread@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.10.3.tgz#b8d0d22f70afa34ad84b7a200ff772f9b9fce474" + integrity sha512-ZZh5leCIlH9lni5bU/wB/UcjtcVLgR8gc+FAgW2OOY+m9h1II3ItTO1/cewNUcsIDZSYcSaz/rYVls+Fb0ExVQ== dependencies: - "@babel/helper-plugin-utils" "^7.10.1" + "@babel/helper-plugin-utils" "^7.10.3" "@babel/plugin-syntax-object-rest-spread" "^7.8.0" "@babel/plugin-transform-parameters" "^7.10.1" @@ -358,12 +358,12 @@ "@babel/helper-plugin-utils" "^7.10.1" "@babel/plugin-syntax-optional-catch-binding" "^7.8.0" -"@babel/plugin-proposal-optional-chaining@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.10.1.tgz#15f5d6d22708629451a91be28f8facc55b0e818c" - integrity sha512-dqQj475q8+/avvok72CF3AOSV/SGEcH29zT5hhohqqvvZ2+boQoOr7iGldBG5YXTO2qgCgc2B3WvVLUdbeMlGA== +"@babel/plugin-proposal-optional-chaining@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.10.3.tgz#9a726f94622b653c0a3a7a59cdce94730f526f7c" + integrity sha512-yyG3n9dJ1vZ6v5sfmIlMMZ8azQoqx/5/nZTSWX1td6L1H1bsjzA8TInDChpafCZiJkeOFzp/PtrfigAQXxI1Ng== dependencies: - "@babel/helper-plugin-utils" "^7.10.1" + "@babel/helper-plugin-utils" "^7.10.3" "@babel/plugin-syntax-optional-chaining" "^7.8.0" "@babel/plugin-proposal-private-methods@^7.10.1": @@ -504,26 +504,26 @@ "@babel/helper-plugin-utils" "^7.10.1" lodash "^4.17.13" -"@babel/plugin-transform-classes@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-classes/-/plugin-transform-classes-7.10.1.tgz#6e11dd6c4dfae70f540480a4702477ed766d733f" - integrity sha512-P9V0YIh+ln/B3RStPoXpEQ/CoAxQIhRSUn7aXqQ+FZJ2u8+oCtjIXR3+X0vsSD8zv+mb56K7wZW1XiDTDGiDRQ== +"@babel/plugin-transform-classes@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-classes/-/plugin-transform-classes-7.10.3.tgz#8d9a656bc3d01f3ff69e1fccb354b0f9d72ac544" + integrity sha512-irEX0ChJLaZVC7FvvRoSIxJlmk0IczFLcwaRXUArBKYHCHbOhe57aG8q3uw/fJsoSXvZhjRX960hyeAGlVBXZw== dependencies: "@babel/helper-annotate-as-pure" "^7.10.1" - "@babel/helper-define-map" "^7.10.1" - "@babel/helper-function-name" "^7.10.1" - "@babel/helper-optimise-call-expression" "^7.10.1" - "@babel/helper-plugin-utils" "^7.10.1" + "@babel/helper-define-map" "^7.10.3" + "@babel/helper-function-name" "^7.10.3" + "@babel/helper-optimise-call-expression" "^7.10.3" + "@babel/helper-plugin-utils" "^7.10.3" "@babel/helper-replace-supers" "^7.10.1" "@babel/helper-split-export-declaration" "^7.10.1" globals "^11.1.0" -"@babel/plugin-transform-computed-properties@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.10.1.tgz#59aa399064429d64dce5cf76ef9b90b7245ebd07" - integrity sha512-mqSrGjp3IefMsXIenBfGcPXxJxweQe2hEIwMQvjtiDQ9b1IBvDUjkAtV/HMXX47/vXf14qDNedXsIiNd1FmkaQ== +"@babel/plugin-transform-computed-properties@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.10.3.tgz#d3aa6eef67cb967150f76faff20f0abbf553757b" + integrity sha512-GWzhaBOsdbjVFav96drOz7FzrcEW6AP5nax0gLIpstiFaI3LOb2tAg06TimaWU6YKOfUACK3FVrxPJ4GSc5TgA== dependencies: - "@babel/helper-plugin-utils" "^7.10.1" + "@babel/helper-plugin-utils" "^7.10.3" "@babel/plugin-transform-destructuring@^7.10.1": version "7.10.1" @@ -603,14 +603,14 @@ "@babel/helper-simple-access" "^7.10.1" babel-plugin-dynamic-import-node "^2.3.3" -"@babel/plugin-transform-modules-systemjs@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.10.1.tgz#9962e4b0ac6aaf2e20431ada3d8ec72082cbffb6" - integrity sha512-ewNKcj1TQZDL3YnO85qh9zo1YF1CHgmSTlRQgHqe63oTrMI85cthKtZjAiZSsSNjPQ5NCaYo5QkbYqEw1ZBgZA== +"@babel/plugin-transform-modules-systemjs@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.10.3.tgz#004ae727b122b7b146b150d50cba5ffbff4ac56b" + integrity sha512-GWXWQMmE1GH4ALc7YXW56BTh/AlzvDWhUNn9ArFF0+Cz5G8esYlVbXfdyHa1xaD1j+GnBoCeoQNlwtZTVdiG/A== dependencies: - "@babel/helper-hoist-variables" "^7.10.1" + "@babel/helper-hoist-variables" "^7.10.3" "@babel/helper-module-transforms" "^7.10.1" - "@babel/helper-plugin-utils" "^7.10.1" + "@babel/helper-plugin-utils" "^7.10.3" babel-plugin-dynamic-import-node "^2.3.3" "@babel/plugin-transform-modules-umd@^7.10.1": @@ -621,10 +621,10 @@ "@babel/helper-module-transforms" "^7.10.1" "@babel/helper-plugin-utils" "^7.10.1" -"@babel/plugin-transform-named-capturing-groups-regex@^7.8.3": - version "7.8.3" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.8.3.tgz#a2a72bffa202ac0e2d0506afd0939c5ecbc48c6c" - integrity sha512-f+tF/8UVPU86TrCb06JoPWIdDpTNSGGcAtaD9mLP0aYGA0OS0j7j7DHJR0GTFrUZPUU6loZhbsVZgTh0N+Qdnw== +"@babel/plugin-transform-named-capturing-groups-regex@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.10.3.tgz#a4f8444d1c5a46f35834a410285f2c901c007ca6" + integrity sha512-I3EH+RMFyVi8Iy/LekQm948Z4Lz4yKT7rK+vuCAeRm0kTa6Z5W7xuhRxDNJv0FPya/her6AUgrDITb70YHtTvA== dependencies: "@babel/helper-create-regexp-features-plugin" "^7.8.3" @@ -658,10 +658,10 @@ dependencies: "@babel/helper-plugin-utils" "^7.10.1" -"@babel/plugin-transform-regenerator@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.10.1.tgz#10e175cbe7bdb63cc9b39f9b3f823c5c7c5c5490" - integrity sha512-B3+Y2prScgJ2Bh/2l9LJxKbb8C8kRfsG4AdPT+n7ixBHIxJaIG8bi8tgjxUMege1+WqSJ+7gu1YeoMVO3gPWzw== +"@babel/plugin-transform-regenerator@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.10.3.tgz#6ec680f140a5ceefd291c221cb7131f6d7e8cb6d" + integrity sha512-H5kNeW0u8mbk0qa1jVIVTeJJL6/TJ81ltD4oyPx0P499DhMJrTmmIFCmJ3QloGpQG8K9symccB7S7SJpCKLwtw== dependencies: regenerator-transform "^0.14.2" @@ -694,13 +694,13 @@ "@babel/helper-plugin-utils" "^7.10.1" "@babel/helper-regex" "^7.10.1" -"@babel/plugin-transform-template-literals@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.10.1.tgz#914c7b7f4752c570ea00553b4284dad8070e8628" - integrity sha512-t7B/3MQf5M1T9hPCRG28DNGZUuxAuDqLYS03rJrIk2prj/UV7Z6FOneijhQhnv/Xa039vidXeVbvjK2SK5f7Gg== +"@babel/plugin-transform-template-literals@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.10.3.tgz#69d39b3d44b31e7b4864173322565894ce939b25" + integrity sha512-yaBn9OpxQra/bk0/CaA4wr41O0/Whkg6nqjqApcinxM7pro51ojhX6fv1pimAnVjVfDy14K0ULoRL70CA9jWWA== dependencies: "@babel/helper-annotate-as-pure" "^7.10.1" - "@babel/helper-plugin-utils" "^7.10.1" + "@babel/helper-plugin-utils" "^7.10.3" "@babel/plugin-transform-typeof-symbol@^7.10.1": version "7.10.1" @@ -733,24 +733,24 @@ "@babel/helper-create-regexp-features-plugin" "^7.10.1" "@babel/helper-plugin-utils" "^7.10.1" -"@babel/preset-env@7.10.2": - version "7.10.2" - resolved "https://registry.yarnpkg.com/@babel/preset-env/-/preset-env-7.10.2.tgz#715930f2cf8573b0928005ee562bed52fb65fdfb" - integrity sha512-MjqhX0RZaEgK/KueRzh+3yPSk30oqDKJ5HP5tqTSB1e2gzGS3PLy7K0BIpnp78+0anFuSwOeuCf1zZO7RzRvEA== +"@babel/preset-env@7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/preset-env/-/preset-env-7.10.3.tgz#3e58c9861bbd93b6a679987c7e4bd365c56c80c9" + integrity sha512-jHaSUgiewTmly88bJtMHbOd1bJf2ocYxb5BWKSDQIP5tmgFuS/n0gl+nhSrYDhT33m0vPxp+rP8oYYgPgMNQlg== dependencies: - "@babel/compat-data" "^7.10.1" + "@babel/compat-data" "^7.10.3" "@babel/helper-compilation-targets" "^7.10.2" - "@babel/helper-module-imports" "^7.10.1" - "@babel/helper-plugin-utils" "^7.10.1" - "@babel/plugin-proposal-async-generator-functions" "^7.10.1" + "@babel/helper-module-imports" "^7.10.3" + "@babel/helper-plugin-utils" "^7.10.3" + "@babel/plugin-proposal-async-generator-functions" "^7.10.3" "@babel/plugin-proposal-class-properties" "^7.10.1" "@babel/plugin-proposal-dynamic-import" "^7.10.1" "@babel/plugin-proposal-json-strings" "^7.10.1" "@babel/plugin-proposal-nullish-coalescing-operator" "^7.10.1" "@babel/plugin-proposal-numeric-separator" "^7.10.1" - "@babel/plugin-proposal-object-rest-spread" "^7.10.1" + "@babel/plugin-proposal-object-rest-spread" "^7.10.3" "@babel/plugin-proposal-optional-catch-binding" "^7.10.1" - "@babel/plugin-proposal-optional-chaining" "^7.10.1" + "@babel/plugin-proposal-optional-chaining" "^7.10.3" "@babel/plugin-proposal-private-methods" "^7.10.1" "@babel/plugin-proposal-unicode-property-regex" "^7.10.1" "@babel/plugin-syntax-async-generators" "^7.8.0" @@ -767,8 +767,8 @@ "@babel/plugin-transform-async-to-generator" "^7.10.1" "@babel/plugin-transform-block-scoped-functions" "^7.10.1" "@babel/plugin-transform-block-scoping" "^7.10.1" - "@babel/plugin-transform-classes" "^7.10.1" - "@babel/plugin-transform-computed-properties" "^7.10.1" + "@babel/plugin-transform-classes" "^7.10.3" + "@babel/plugin-transform-computed-properties" "^7.10.3" "@babel/plugin-transform-destructuring" "^7.10.1" "@babel/plugin-transform-dotall-regex" "^7.10.1" "@babel/plugin-transform-duplicate-keys" "^7.10.1" @@ -779,24 +779,24 @@ "@babel/plugin-transform-member-expression-literals" "^7.10.1" "@babel/plugin-transform-modules-amd" "^7.10.1" "@babel/plugin-transform-modules-commonjs" "^7.10.1" - "@babel/plugin-transform-modules-systemjs" "^7.10.1" + "@babel/plugin-transform-modules-systemjs" "^7.10.3" "@babel/plugin-transform-modules-umd" "^7.10.1" - "@babel/plugin-transform-named-capturing-groups-regex" "^7.8.3" + "@babel/plugin-transform-named-capturing-groups-regex" "^7.10.3" "@babel/plugin-transform-new-target" "^7.10.1" "@babel/plugin-transform-object-super" "^7.10.1" "@babel/plugin-transform-parameters" "^7.10.1" "@babel/plugin-transform-property-literals" "^7.10.1" - "@babel/plugin-transform-regenerator" "^7.10.1" + "@babel/plugin-transform-regenerator" "^7.10.3" "@babel/plugin-transform-reserved-words" "^7.10.1" "@babel/plugin-transform-shorthand-properties" "^7.10.1" "@babel/plugin-transform-spread" "^7.10.1" "@babel/plugin-transform-sticky-regex" "^7.10.1" - "@babel/plugin-transform-template-literals" "^7.10.1" + "@babel/plugin-transform-template-literals" "^7.10.3" "@babel/plugin-transform-typeof-symbol" "^7.10.1" "@babel/plugin-transform-unicode-escapes" "^7.10.1" "@babel/plugin-transform-unicode-regex" "^7.10.1" "@babel/preset-modules" "^0.1.3" - "@babel/types" "^7.10.2" + "@babel/types" "^7.10.3" browserslist "^4.12.0" core-js-compat "^3.6.2" invariant "^2.2.2" @@ -822,10 +822,10 @@ "@babel/helper-plugin-utils" "^7.10.1" "@babel/plugin-transform-typescript" "^7.10.1" -"@babel/register@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/register/-/register-7.10.1.tgz#b6567c5cb5049f44bbf8c35d6ff68ca3c43238ed" - integrity sha512-sl96+kB3IA2B9EzpwwBmYadOT14vw3KaXOknGDbJaZCOj52GDA4Tivudq9doCJcB+bEIKCEARZYwRgBBsCGXyg== +"@babel/register@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/register/-/register-7.10.3.tgz#b49b6603fc8d214cd2f77a6ed2256bd198b5994b" + integrity sha512-s1il0vdd02HCGwV1iocGJEzcbTNouZqMolSXKXFAiTNJSudPas9jdLQwyPPyAJxdNL6KGJ8pwWIOpKmgO/JWqg== dependencies: find-cache-dir "^2.0.0" lodash "^4.17.13" @@ -840,36 +840,36 @@ dependencies: regenerator-runtime "^0.13.4" -"@babel/template@^7.10.1", "@babel/template@^7.3.3": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.10.1.tgz#e167154a94cb5f14b28dc58f5356d2162f539811" - integrity sha512-OQDg6SqvFSsc9A0ej6SKINWrpJiNonRIniYondK2ViKhB06i3c0s+76XUft71iqBEe9S1OKsHwPAjfHnuvnCig== +"@babel/template@^7.10.1", "@babel/template@^7.10.3", "@babel/template@^7.3.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.10.3.tgz#4d13bc8e30bf95b0ce9d175d30306f42a2c9a7b8" + integrity sha512-5BjI4gdtD+9fHZUsaxPHPNpwa+xRkDO7c7JbhYn2afvrkDu5SfAAbi9AIMXw2xEhO/BR35TqiW97IqNvCo/GqA== dependencies: - "@babel/code-frame" "^7.10.1" - "@babel/parser" "^7.10.1" - "@babel/types" "^7.10.1" + "@babel/code-frame" "^7.10.3" + "@babel/parser" "^7.10.3" + "@babel/types" "^7.10.3" -"@babel/traverse@^7.1.0", "@babel/traverse@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.10.1.tgz#bbcef3031e4152a6c0b50147f4958df54ca0dd27" - integrity sha512-C/cTuXeKt85K+p08jN6vMDz8vSV0vZcI0wmQ36o6mjbuo++kPMdpOYw23W2XH04dbRt9/nMEfA4W3eR21CD+TQ== +"@babel/traverse@^7.1.0", "@babel/traverse@^7.10.1", "@babel/traverse@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.10.3.tgz#0b01731794aa7b77b214bcd96661f18281155d7e" + integrity sha512-qO6623eBFhuPm0TmmrUFMT1FulCmsSeJuVGhiLodk2raUDFhhTECLd9E9jC4LBIWziqt4wgF6KuXE4d+Jz9yug== dependencies: - "@babel/code-frame" "^7.10.1" - "@babel/generator" "^7.10.1" - "@babel/helper-function-name" "^7.10.1" + "@babel/code-frame" "^7.10.3" + "@babel/generator" "^7.10.3" + "@babel/helper-function-name" "^7.10.3" "@babel/helper-split-export-declaration" "^7.10.1" - "@babel/parser" "^7.10.1" - "@babel/types" "^7.10.1" + "@babel/parser" "^7.10.3" + "@babel/types" "^7.10.3" debug "^4.1.0" globals "^11.1.0" lodash "^4.17.13" -"@babel/types@^7.0.0", "@babel/types@^7.10.1", "@babel/types@^7.10.2", "@babel/types@^7.3.0", "@babel/types@^7.3.3", "@babel/types@^7.4.4": - version "7.10.2" - resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.10.2.tgz#30283be31cad0dbf6fb00bd40641ca0ea675172d" - integrity sha512-AD3AwWBSz0AWF0AkCN9VPiWrvldXq+/e3cHa4J89vo4ymjz1XwrBFFVZmkJTsQIPNk+ZVomPSXUJqq8yyjZsng== +"@babel/types@^7.0.0", "@babel/types@^7.10.1", "@babel/types@^7.10.3", "@babel/types@^7.3.0", "@babel/types@^7.3.3", "@babel/types@^7.4.4": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.10.3.tgz#6535e3b79fea86a6b09e012ea8528f935099de8e" + integrity sha512-nZxaJhBXBQ8HVoIcGsf9qWep3Oh3jCENK54V4mRF7qaJabVsAYdbTtmSD8WmAp1R6ytPiu5apMwSXyxB1WlaBA== dependencies: - "@babel/helper-validator-identifier" "^7.10.1" + "@babel/helper-validator-identifier" "^7.10.3" lodash "^4.17.13" to-fast-properties "^2.0.0" From daf2a485abebfef016357f57bf55b886b361b706 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Tue, 23 Jun 2020 09:25:03 +0200 Subject: [PATCH 09/25] feat: abortOnError / abortIgnoreStatusCodes (#6556) --- docs/usage/configuration-options.md | 61 +++++++++++++++++++++++++++++ lib/config/definitions.ts | 34 ++++++++++++++-- lib/types/host-rules.ts | 2 + lib/util/http/host-rules.ts | 11 ++++-- lib/util/http/index.spec.ts | 26 ++++++++++++ lib/util/http/index.ts | 22 +++++++++-- 6 files changed, 146 insertions(+), 10 deletions(-) diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index 309b4a2217dbef..236305f98eb85d 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -445,6 +445,67 @@ Example for configuring `docker` auth: } ``` +### abortIgnoreStatusCodes + +This field can be used to configure status codes that Renovate ignores and passes through when `abortOnError` is set to `true`. For example to also skip 404 responses then configure the following: + +```json +{ + "hostRules": [ + { + "abortOnError": true, + "abortStatusCodes": [404] + } + ] +} +``` + +Note that this field is _not_ mergeable, so the last-applied host rule will take precedence. + +### abortOnError + +Use this field to configure Renovate to abort runs for custom hosts. By default, Renovate will only abort for known public hosts, which has the downside that transient errors for other hosts can cause autoclosing of PRs. + +To abort Renovate runs for http failures from _any_ host: + +```json +{ + "hostRules": [ + { + "abortOnError": true + } + ] +} +``` + +To abort Renovate runs for any `docker` datasource failures: + +```json +{ + "hostRules": [ + { + "hostType": "docker", + "abortOnError": true + } + ] +} +``` + +To abort Renovate for errors for a specific `docker` host: + +```json +{ + "hostRules": [ + { + "hostName": "docker.company.com", + "abortOnError": true + } + ] +} +``` + +When this field is enabled, Renovate will abort its run if it encounters either (a) any low-level http error (e.g. `ETIMEDOUT`) or (b) receives a response _not_ matching any of the configured `abortIgnoreStatusCodes` (e.g. `500 Internal Error`); + ### baseUrl Use this instead of `domainName` or `hostName` if you need a rule to apply to a specific path on a host. For example, `"baseUrl": "https://api.github.com"` is equivalent to `"hostName": "api.github.com"` but `"baseUrl": "https://api.github.com/google/"` is not. diff --git a/lib/config/definitions.ts b/lib/config/definitions.ts index df4622ca1803e8..ca7e6d77a4848e 100644 --- a/lib/config/definitions.ts +++ b/lib/config/definitions.ts @@ -38,12 +38,13 @@ export interface RenovateOptionBase { stage?: RenovateConfigStage; } -export interface RenovateArrayOption - extends RenovateOptionBase { +export interface RenovateArrayOption< + T extends string | number | object = object +> extends RenovateOptionBase { default?: T[]; mergeable?: boolean; type: 'array'; - subType?: 'string' | 'object'; + subType?: 'string' | 'object' | 'number'; } export interface RenovateStringArrayOption extends RenovateArrayOption { @@ -51,6 +52,10 @@ export interface RenovateStringArrayOption extends RenovateArrayOption { subType: 'string'; } +export interface RenovateNumberArrayOption extends RenovateArrayOption { + subType: 'number'; +} + export interface RenovateBooleanOption extends RenovateOptionBase { default?: boolean; type: 'boolean'; @@ -79,6 +84,7 @@ export interface RenovateObjectOption extends RenovateOptionBase { export type RenovateOptions = | RenovateStringOption + | RenovateNumberArrayOption | RenovateStringArrayOption | RenovateIntegerOption | RenovateBooleanOption @@ -1613,6 +1619,28 @@ const options: RenovateOptions[] = [ cli: false, env: false, }, + { + name: 'abortOnError', + description: + 'If enabled, Renovate will abort its run when http request errors occur.', + type: 'boolean', + stage: 'repository', + parent: 'hostRules', + default: false, + cli: false, + env: false, + }, + { + name: 'abortIgnoreStatusCodes', + description: + 'A list of HTTP status codes to ignore and *not* abort the run because of when abortOnError=true.', + type: 'array', + subType: 'number', + stage: 'repository', + parent: 'hostRules', + cli: false, + env: false, + }, { name: 'prBodyDefinitions', description: 'Table column definitions for use in PR tables', diff --git a/lib/types/host-rules.ts b/lib/types/host-rules.ts index 96a6a8477d10c0..403b8dad9730c1 100644 --- a/lib/types/host-rules.ts +++ b/lib/types/host-rules.ts @@ -13,4 +13,6 @@ export interface HostRule { platform?: string; timeout?: number; encrypted?: HostRule; + abortOnError?: boolean; + abortIgnoreStatusCodes?: number[]; } diff --git a/lib/util/http/host-rules.ts b/lib/util/http/host-rules.ts index 57c6f4e201d920..6d1f7229893a1d 100644 --- a/lib/util/http/host-rules.ts +++ b/lib/util/http/host-rules.ts @@ -10,7 +10,7 @@ export function applyHostRules(url: string, inOptions: any): any { hostType: options.hostType, url, }) || {}; - const { username, password, token, timeout } = foundRules; + const { username, password, token } = foundRules; if (options.headers?.authorization || options.auth || options.token) { logger.trace('Authorization already set for host: ' + options.hostname); } else if (password) { @@ -20,8 +20,11 @@ export function applyHostRules(url: string, inOptions: any): any { logger.trace('Applying Bearer authentication for host ' + options.hostname); options.token = token; } - if (timeout) { - options.timeout = timeout; - } + // Apply optional params + ['abortOnError', 'abortIgnoreStatusCodes', 'timeout'].forEach((param) => { + if (foundRules[param]) { + options[param] = foundRules[param]; + } + }); return options; } diff --git a/lib/util/http/index.spec.ts b/lib/util/http/index.spec.ts index 78dfc1938c75f2..f97e240dc77c21 100644 --- a/lib/util/http/index.spec.ts +++ b/lib/util/http/index.spec.ts @@ -1,5 +1,7 @@ import nock from 'nock'; import { getName } from '../../../test/util'; +import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages'; +import * as hostRules from '../host-rules'; import { Http } from '.'; const baseUrl = 'http://renovate.com'; @@ -10,12 +12,36 @@ describe(getName(__filename), () => { beforeEach(() => { http = new Http('dummy'); nock.cleanAll(); + hostRules.clear(); }); it('get', async () => { nock(baseUrl).get('/test').reply(200); expect(await http.get('http://renovate.com/test')).toMatchSnapshot(); expect(nock.isDone()).toBe(true); }); + it('returns 429 error', async () => { + nock(baseUrl).get('/test').reply(429); + await expect(http.get('http://renovate.com/test')).rejects.toThrow( + 'Response code 429 (Too Many Requests)' + ); + expect(nock.isDone()).toBe(true); + }); + it('converts 404 error to ExternalHostError', async () => { + nock(baseUrl).get('/test').reply(404); + hostRules.add({ abortOnError: true }); + await expect(http.get('http://renovate.com/test')).rejects.toThrow( + EXTERNAL_HOST_ERROR + ); + expect(nock.isDone()).toBe(true); + }); + it('ignores 404 error and does not throw ExternalHostError', async () => { + nock(baseUrl).get('/test').reply(404); + hostRules.add({ abortOnError: true, abortIgnoreStatusCodes: [404] }); + await expect(http.get('http://renovate.com/test')).rejects.toThrow( + 'Response code 404 (Not Found)' + ); + expect(nock.isDone()).toBe(true); + }); it('getJson', async () => { nock(baseUrl).get('/').reply(200, '{ "test": true }'); expect(await http.getJson('http://renovate.com')).toMatchSnapshot(); diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts index 5b456e3bee2bff..bb42c5b888d677 100644 --- a/lib/util/http/index.ts +++ b/lib/util/http/index.ts @@ -1,6 +1,7 @@ import crypto from 'crypto'; import URL from 'url'; import got from 'got'; +import { ExternalHostError } from '../../types/error'; import * as runCache from '../cache/run'; import { clone } from '../clone'; import { applyAuthorization } from './auth'; @@ -41,6 +42,21 @@ function cloneResponse(response: any): HttpResponse { }; } +async function resolveResponse( + promisedRes: Promise>, + { abortOnError, abortIgnoreStatusCodes } +): Promise> { + try { + const res = await promisedRes; + return cloneResponse(res); + } catch (err) { + if (abortOnError && !abortIgnoreStatusCodes?.includes(err.statusCode)) { + throw new ExternalHostError(err); + } + throw err; + } +} + export class Http { constructor(private hostType: string, private options?: HttpOptions) {} @@ -102,7 +118,7 @@ export class Http { const cachedRes = runCache.get(cacheKey); // istanbul ignore if if (cachedRes) { - return cloneResponse(await cachedRes); + return resolveResponse(cachedRes, options); } } const startTime = Date.now(); @@ -110,7 +126,7 @@ export class Http { if (options.method === 'get') { runCache.set(cacheKey, promisedRes); // always set if it's a get } - const res = await promisedRes; + const res = await resolveResponse(promisedRes, options); const httpRequests = runCache.get('http-requests') || []; httpRequests.push({ method: options.method, @@ -118,7 +134,7 @@ export class Http { duration: Date.now() - startTime, }); runCache.set('http-requests', httpRequests); - return cloneResponse(res); + return res; } get(url: string, options: HttpOptions = {}): Promise { From 031ad87b62e8acb24f1b6b567cd6cf9510b41b8a Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Tue, 23 Jun 2020 11:44:52 +0200 Subject: [PATCH 10/25] refactor: move ExternalHostError --- lib/config/presets/github/index.ts | 2 +- lib/config/presets/gitlab/index.ts | 2 +- lib/config/presets/index.ts | 2 +- lib/constants/error-messages.ts | 1 + lib/datasource/cdnjs/index.ts | 2 +- lib/datasource/crate/index.ts | 2 +- lib/datasource/dart/index.ts | 2 +- lib/datasource/docker/index.ts | 2 +- lib/datasource/galaxy/index.ts | 2 +- lib/datasource/gradle-version/index.ts | 2 +- lib/datasource/helm/index.ts | 2 +- lib/datasource/hex/index.ts | 2 +- lib/datasource/index.spec.ts | 2 +- lib/datasource/index.ts | 2 +- lib/datasource/maven/util.ts | 2 +- lib/datasource/npm/get.spec.ts | 2 +- lib/datasource/npm/get.ts | 2 +- lib/datasource/packagist/index.ts | 2 +- lib/datasource/pod/index.ts | 2 +- lib/datasource/repology/index.ts | 2 +- lib/datasource/ruby-version/index.ts | 2 +- lib/datasource/rubygems/get-rubygems-org.ts | 2 +- lib/datasource/terraform-module/index.ts | 2 +- lib/manager/gradle/index.ts | 2 +- lib/manager/npm/post-update/index.ts | 2 +- lib/manager/npm/post-update/yarn.ts | 2 +- lib/platform/git/storage.ts | 2 +- lib/platform/github/index.ts | 2 +- lib/types/{error.ts => errors/external-host-error.ts} | 2 +- lib/util/http/github.ts | 2 +- lib/util/http/gitlab.ts | 2 +- lib/util/http/index.ts | 2 +- lib/workers/branch/index.ts | 2 +- lib/workers/pr/index.ts | 2 +- lib/workers/repository/error.spec.ts | 2 +- lib/workers/repository/error.ts | 2 +- lib/workers/repository/init/config.ts | 2 +- 37 files changed, 37 insertions(+), 36 deletions(-) rename lib/types/{error.ts => errors/external-host-error.ts} (87%) diff --git a/lib/config/presets/github/index.ts b/lib/config/presets/github/index.ts index 2a781f073aea6e..17cddca24dc091 100644 --- a/lib/config/presets/github/index.ts +++ b/lib/config/presets/github/index.ts @@ -1,6 +1,6 @@ import { PLATFORM_TYPE_GITHUB } from '../../../constants/platforms'; import { logger } from '../../../logger'; -import { ExternalHostError } from '../../../types/error'; +import { ExternalHostError } from '../../../types/errors/external-host-error'; import { Http, HttpOptions } from '../../../util/http'; import { Preset, PresetConfig } from '../common'; import { PRESET_DEP_NOT_FOUND, fetchPreset } from '../util'; diff --git a/lib/config/presets/gitlab/index.ts b/lib/config/presets/gitlab/index.ts index a5f7eb78211cf1..0e7f9d0c93fdba 100644 --- a/lib/config/presets/gitlab/index.ts +++ b/lib/config/presets/gitlab/index.ts @@ -1,5 +1,5 @@ import { logger } from '../../../logger'; -import { ExternalHostError } from '../../../types/error'; +import { ExternalHostError } from '../../../types/errors/external-host-error'; import type { GitLabBranch } from '../../../types/platform/gitlab'; import { GitlabHttp } from '../../../util/http/gitlab'; import { Preset, PresetConfig } from '../common'; diff --git a/lib/config/presets/index.ts b/lib/config/presets/index.ts index 3a9eea8fdfe6d1..249df7e555c6e5 100644 --- a/lib/config/presets/index.ts +++ b/lib/config/presets/index.ts @@ -1,7 +1,7 @@ import is from '@sindresorhus/is'; import { CONFIG_VALIDATION } from '../../constants/error-messages'; import { logger } from '../../logger'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import { regEx } from '../../util/regex'; import { RenovateConfig } from '../common'; import * as massage from '../massage'; diff --git a/lib/constants/error-messages.ts b/lib/constants/error-messages.ts index 12ad1164c19456..fa4f6e4e307b9b 100644 --- a/lib/constants/error-messages.ts +++ b/lib/constants/error-messages.ts @@ -35,6 +35,7 @@ export const MANAGER_NO_PACKAGE_FILES = 'no-package-files'; // Host error export const EXTERNAL_HOST_ERROR = 'external-host-error'; +export const IGNORABLE_HOST_ERROR = 'ignorable-host-error'; // Worker Error export const WORKER_FILE_UPDATE_FAILED = 'update-failure'; diff --git a/lib/datasource/cdnjs/index.ts b/lib/datasource/cdnjs/index.ts index c408bd0d147e71..a994a6b8c639e8 100644 --- a/lib/datasource/cdnjs/index.ts +++ b/lib/datasource/cdnjs/index.ts @@ -1,5 +1,5 @@ import { logger } from '../../logger'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import { Http } from '../../util/http'; import { CachePromise, cacheAble } from '../cache'; import { GetReleasesConfig, ReleaseResult } from '../common'; diff --git a/lib/datasource/crate/index.ts b/lib/datasource/crate/index.ts index 40134c936da8ca..13d2c000d23652 100644 --- a/lib/datasource/crate/index.ts +++ b/lib/datasource/crate/index.ts @@ -1,5 +1,5 @@ import { logger } from '../../logger'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import * as globalCache from '../../util/cache/global'; import { Http } from '../../util/http'; import { GetReleasesConfig, Release, ReleaseResult } from '../common'; diff --git a/lib/datasource/dart/index.ts b/lib/datasource/dart/index.ts index 4043f1df5b3248..241394373b5d7b 100644 --- a/lib/datasource/dart/index.ts +++ b/lib/datasource/dart/index.ts @@ -1,5 +1,5 @@ import { logger } from '../../logger'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import { Http, HttpResponse } from '../../util/http'; import { GetReleasesConfig, ReleaseResult } from '../common'; diff --git a/lib/datasource/docker/index.ts b/lib/datasource/docker/index.ts index 7a9e57c4199f41..a295738843c9e3 100644 --- a/lib/datasource/docker/index.ts +++ b/lib/datasource/docker/index.ts @@ -6,7 +6,7 @@ import parseLinkHeader from 'parse-link-header'; import wwwAuthenticate from 'www-authenticate'; import { logger } from '../../logger'; import { HostRule } from '../../types'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import * as globalCache from '../../util/cache/global'; import * as hostRules from '../../util/host-rules'; import { Http, HttpResponse } from '../../util/http'; diff --git a/lib/datasource/galaxy/index.ts b/lib/datasource/galaxy/index.ts index a76942dfd4c43f..14c67e18832feb 100644 --- a/lib/datasource/galaxy/index.ts +++ b/lib/datasource/galaxy/index.ts @@ -1,5 +1,5 @@ import { logger } from '../../logger'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import * as globalCache from '../../util/cache/global'; import { Http } from '../../util/http'; import { GetReleasesConfig, Release, ReleaseResult } from '../common'; diff --git a/lib/datasource/gradle-version/index.ts b/lib/datasource/gradle-version/index.ts index 624bb27b5ea89e..f7ebc86dac1b2f 100644 --- a/lib/datasource/gradle-version/index.ts +++ b/lib/datasource/gradle-version/index.ts @@ -1,5 +1,5 @@ import { logger } from '../../logger'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import { Http } from '../../util/http'; import { regEx } from '../../util/regex'; import { GetReleasesConfig, ReleaseResult } from '../common'; diff --git a/lib/datasource/helm/index.ts b/lib/datasource/helm/index.ts index 5ba8ba238fa10e..24a1910d78937b 100644 --- a/lib/datasource/helm/index.ts +++ b/lib/datasource/helm/index.ts @@ -1,7 +1,7 @@ import yaml from 'js-yaml'; import { logger } from '../../logger'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import * as globalCache from '../../util/cache/global'; import { Http } from '../../util/http'; import { ensureTrailingSlash } from '../../util/url'; diff --git a/lib/datasource/hex/index.ts b/lib/datasource/hex/index.ts index c619cbffc0ba13..e0ce145c7b0868 100644 --- a/lib/datasource/hex/index.ts +++ b/lib/datasource/hex/index.ts @@ -1,5 +1,5 @@ import { logger } from '../../logger'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import { Http } from '../../util/http'; import { GetReleasesConfig, ReleaseResult } from '../common'; diff --git a/lib/datasource/index.spec.ts b/lib/datasource/index.spec.ts index 34aa6531a7a9bc..160a5bb30f50a0 100644 --- a/lib/datasource/index.spec.ts +++ b/lib/datasource/index.spec.ts @@ -1,6 +1,6 @@ import { mocked } from '../../test/util'; import { EXTERNAL_HOST_ERROR } from '../constants/error-messages'; -import { ExternalHostError } from '../types/error'; +import { ExternalHostError } from '../types/errors/external-host-error'; import { loadModules } from '../util/modules'; import * as datasourceDocker from './docker'; import * as datasourceGithubTags from './github-tags'; diff --git a/lib/datasource/index.ts b/lib/datasource/index.ts index dba7f32e5fef06..63e982127efa6e 100644 --- a/lib/datasource/index.ts +++ b/lib/datasource/index.ts @@ -1,7 +1,7 @@ import is from '@sindresorhus/is'; import _ from 'lodash'; import { logger } from '../logger'; -import { ExternalHostError } from '../types/error'; +import { ExternalHostError } from '../types/errors/external-host-error'; import * as runCache from '../util/cache/run'; import { clone } from '../util/clone'; import * as allVersioning from '../versioning'; diff --git a/lib/datasource/maven/util.ts b/lib/datasource/maven/util.ts index bb03867e9a390f..b80125d5fb5544 100644 --- a/lib/datasource/maven/util.ts +++ b/lib/datasource/maven/util.ts @@ -1,6 +1,6 @@ import url from 'url'; import { logger } from '../../logger'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import { Http } from '../../util/http'; import { MAVEN_REPO, id } from './common'; diff --git a/lib/datasource/npm/get.spec.ts b/lib/datasource/npm/get.spec.ts index ec305f257982ec..fb80cbfe541dd4 100644 --- a/lib/datasource/npm/get.spec.ts +++ b/lib/datasource/npm/get.spec.ts @@ -1,6 +1,6 @@ import * as httpMock from '../../../test/httpMock'; import { getName } from '../../../test/util'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import { getDependency, resetMemCache } from './get'; import { setNpmrc } from './npmrc'; diff --git a/lib/datasource/npm/get.ts b/lib/datasource/npm/get.ts index dee919732af7e7..2dc1a9047dec7e 100644 --- a/lib/datasource/npm/get.ts +++ b/lib/datasource/npm/get.ts @@ -6,7 +6,7 @@ import moment from 'moment'; import registryAuthToken from 'registry-auth-token'; import getRegistryUrl from 'registry-auth-token/registry-url'; import { logger } from '../../logger'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import * as globalCache from '../../util/cache/global'; import { find } from '../../util/host-rules'; import { Http, HttpOptions } from '../../util/http'; diff --git a/lib/datasource/packagist/index.ts b/lib/datasource/packagist/index.ts index 7fce259e40b1d5..e0cdfc57ec694c 100644 --- a/lib/datasource/packagist/index.ts +++ b/lib/datasource/packagist/index.ts @@ -2,7 +2,7 @@ import URL from 'url'; import pAll from 'p-all'; import { logger } from '../../logger'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import * as globalCache from '../../util/cache/global'; import * as runCache from '../../util/cache/run'; import * as hostRules from '../../util/host-rules'; diff --git a/lib/datasource/pod/index.ts b/lib/datasource/pod/index.ts index 391719f73187cb..00b6853cabbc28 100644 --- a/lib/datasource/pod/index.ts +++ b/lib/datasource/pod/index.ts @@ -1,6 +1,6 @@ import crypto from 'crypto'; import { logger } from '../../logger'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import * as globalCache from '../../util/cache/global'; import { Http } from '../../util/http'; import { GithubHttp } from '../../util/http/github'; diff --git a/lib/datasource/repology/index.ts b/lib/datasource/repology/index.ts index 7af17eb7753cff..784b72952e3c07 100644 --- a/lib/datasource/repology/index.ts +++ b/lib/datasource/repology/index.ts @@ -1,6 +1,6 @@ import { URLSearchParams } from 'url'; import { logger } from '../../logger'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import * as globalCache from '../../util/cache/global'; import { Http } from '../../util/http'; import { GetReleasesConfig, ReleaseResult } from '../common'; diff --git a/lib/datasource/ruby-version/index.ts b/lib/datasource/ruby-version/index.ts index d85ee6cec9d591..433ede5c0612cd 100644 --- a/lib/datasource/ruby-version/index.ts +++ b/lib/datasource/ruby-version/index.ts @@ -1,6 +1,6 @@ import { parse } from 'node-html-parser'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import * as globalCache from '../../util/cache/global'; import { Http } from '../../util/http'; import { isVersion } from '../../versioning/ruby'; diff --git a/lib/datasource/rubygems/get-rubygems-org.ts b/lib/datasource/rubygems/get-rubygems-org.ts index f3005573227fa5..05dbf41e5abb1c 100644 --- a/lib/datasource/rubygems/get-rubygems-org.ts +++ b/lib/datasource/rubygems/get-rubygems-org.ts @@ -1,5 +1,5 @@ import { logger } from '../../logger'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import { Http } from '../../util/http'; import { ReleaseResult } from '../common'; import { id } from './common'; diff --git a/lib/datasource/terraform-module/index.ts b/lib/datasource/terraform-module/index.ts index 4cac9a7d805a35..5747efb1110730 100644 --- a/lib/datasource/terraform-module/index.ts +++ b/lib/datasource/terraform-module/index.ts @@ -1,5 +1,5 @@ import { logger } from '../../logger'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import * as globalCache from '../../util/cache/global'; import { Http } from '../../util/http'; import { GetReleasesConfig, ReleaseResult } from '../common'; diff --git a/lib/manager/gradle/index.ts b/lib/manager/gradle/index.ts index 2b1d02d87b1946..2af2a8e97e5492 100644 --- a/lib/manager/gradle/index.ts +++ b/lib/manager/gradle/index.ts @@ -5,7 +5,7 @@ import upath from 'upath'; import { LANGUAGE_JAVA } from '../../constants/languages'; import * as datasourceMaven from '../../datasource/maven'; import { logger } from '../../logger'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import { ExecOptions, exec } from '../../util/exec'; import { BinarySource } from '../../util/exec/common'; import { readLocalFile } from '../../util/fs'; diff --git a/lib/manager/npm/post-update/index.ts b/lib/manager/npm/post-update/index.ts index 75cf85f9c3f6c7..f380fb289096ca 100644 --- a/lib/manager/npm/post-update/index.ts +++ b/lib/manager/npm/post-update/index.ts @@ -6,7 +6,7 @@ import { SYSTEM_INSUFFICIENT_DISK_SPACE } from '../../../constants/error-message import { id as npmId } from '../../../datasource/npm'; import { logger } from '../../../logger'; import { platform } from '../../../platform'; -import { ExternalHostError } from '../../../types/error'; +import { ExternalHostError } from '../../../types/errors/external-host-error'; import { getChildProcessEnv } from '../../../util/exec/env'; import { deleteLocalFile } from '../../../util/fs'; import * as hostRules from '../../../util/host-rules'; diff --git a/lib/manager/npm/post-update/yarn.ts b/lib/manager/npm/post-update/yarn.ts index 1dcc798c0b1884..f845e894b30063 100644 --- a/lib/manager/npm/post-update/yarn.ts +++ b/lib/manager/npm/post-update/yarn.ts @@ -6,7 +6,7 @@ import { join } from 'upath'; import { SYSTEM_INSUFFICIENT_DISK_SPACE } from '../../../constants/error-messages'; import { id as npmId } from '../../../datasource/npm'; import { logger } from '../../../logger'; -import { ExternalHostError } from '../../../types/error'; +import { ExternalHostError } from '../../../types/errors/external-host-error'; import { ExecOptions, exec } from '../../../util/exec'; import { PostUpdateConfig, Upgrade } from '../../common'; import { getNodeConstraint } from './node-version'; diff --git a/lib/platform/git/storage.ts b/lib/platform/git/storage.ts index b857ed7f36e753..aa9e3ec0c1ff98 100644 --- a/lib/platform/git/storage.ts +++ b/lib/platform/git/storage.ts @@ -10,7 +10,7 @@ import { SYSTEM_INSUFFICIENT_DISK_SPACE, } from '../../constants/error-messages'; import { logger } from '../../logger'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import * as limits from '../../workers/global/limits'; import { CommitFilesConfig } from '../common'; import { writePrivateKey } from './private-key'; diff --git a/lib/platform/github/index.ts b/lib/platform/github/index.ts index 644643b558675e..d23dc10cbb3ad0 100644 --- a/lib/platform/github/index.ts +++ b/lib/platform/github/index.ts @@ -23,7 +23,7 @@ import { } from '../../constants/pull-requests'; import { logger } from '../../logger'; import { BranchStatus } from '../../types'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import * as hostRules from '../../util/host-rules'; import * as githubHttp from '../../util/http/github'; import { sanitize } from '../../util/sanitize'; diff --git a/lib/types/error.ts b/lib/types/errors/external-host-error.ts similarity index 87% rename from lib/types/error.ts rename to lib/types/errors/external-host-error.ts index b6e67592d0b28a..88f4fdb936f19f 100644 --- a/lib/types/error.ts +++ b/lib/types/errors/external-host-error.ts @@ -1,4 +1,4 @@ -import { EXTERNAL_HOST_ERROR } from '../constants/error-messages'; +import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages'; export class ExternalHostError extends Error { hostType: string; diff --git a/lib/util/http/github.ts b/lib/util/http/github.ts index c2f6dd1b657c28..d1c1fcbf3ad52a 100644 --- a/lib/util/http/github.ts +++ b/lib/util/http/github.ts @@ -10,7 +10,7 @@ import { } from '../../constants/error-messages'; import { PLATFORM_TYPE_GITHUB } from '../../constants/platforms'; import { logger } from '../../logger'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import { maskToken } from '../mask'; import { Http, HttpPostOptions, HttpResponse, InternalHttpOptions } from '.'; diff --git a/lib/util/http/gitlab.ts b/lib/util/http/gitlab.ts index 5e759d333952b2..34b8496285a944 100644 --- a/lib/util/http/gitlab.ts +++ b/lib/util/http/gitlab.ts @@ -2,7 +2,7 @@ import { URL } from 'url'; import parseLinkHeader from 'parse-link-header'; import { PLATFORM_TYPE_GITLAB } from '../../constants/platforms'; import { logger } from '../../logger'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import { Http, HttpResponse, InternalHttpOptions } from '.'; let baseUrl = 'https://gitlab.com/api/v4/'; diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts index bb42c5b888d677..43da6e0cb8b371 100644 --- a/lib/util/http/index.ts +++ b/lib/util/http/index.ts @@ -1,7 +1,7 @@ import crypto from 'crypto'; import URL from 'url'; import got from 'got'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import * as runCache from '../cache/run'; import { clone } from '../clone'; import { applyAuthorization } from './auth'; diff --git a/lib/workers/branch/index.ts b/lib/workers/branch/index.ts index a646d6dc708264..416a2380a36778 100644 --- a/lib/workers/branch/index.ts +++ b/lib/workers/branch/index.ts @@ -22,7 +22,7 @@ import { logger } from '../../logger'; import { getAdditionalFiles } from '../../manager/npm/post-update'; import { platform } from '../../platform'; import { BranchStatus } from '../../types'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import { emojify } from '../../util/emoji'; import { exec } from '../../util/exec'; import { readLocalFile, writeLocalFile } from '../../util/fs'; diff --git a/lib/workers/pr/index.ts b/lib/workers/pr/index.ts index 17c2bcdca22108..d960b93f175b18 100644 --- a/lib/workers/pr/index.ts +++ b/lib/workers/pr/index.ts @@ -9,7 +9,7 @@ import { import { logger } from '../../logger'; import { PlatformPrOptions, Pr, platform } from '../../platform'; import { BranchStatus } from '../../types'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import { BranchConfig, PrResult } from '../common'; import { getPrBody } from './body'; import { ChangeLogError } from './changelog'; diff --git a/lib/workers/repository/error.spec.ts b/lib/workers/repository/error.spec.ts index 5fce970db9260a..29be585ad698ae 100644 --- a/lib/workers/repository/error.spec.ts +++ b/lib/workers/repository/error.spec.ts @@ -26,7 +26,7 @@ import { SYSTEM_INSUFFICIENT_MEMORY, UNKNOWN_ERROR, } from '../../constants/error-messages'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import handleError from './error'; jest.mock('./error-config'); diff --git a/lib/workers/repository/error.ts b/lib/workers/repository/error.ts index 5ff14d0aeac259..3014762e6f9140 100644 --- a/lib/workers/repository/error.ts +++ b/lib/workers/repository/error.ts @@ -28,7 +28,7 @@ import { UNKNOWN_ERROR, } from '../../constants/error-messages'; import { logger } from '../../logger'; -import { ExternalHostError } from '../../types/error'; +import { ExternalHostError } from '../../types/errors/external-host-error'; import { raiseConfigWarningIssue } from './error-config'; export default async function handleError( diff --git a/lib/workers/repository/init/config.ts b/lib/workers/repository/init/config.ts index 21ce3d73254720..2eeb01658605d0 100644 --- a/lib/workers/repository/init/config.ts +++ b/lib/workers/repository/init/config.ts @@ -11,7 +11,7 @@ import { CONFIG_VALIDATION } from '../../../constants/error-messages'; import * as npmApi from '../../../datasource/npm'; import { logger } from '../../../logger'; import { platform } from '../../../platform'; -import { ExternalHostError } from '../../../types/error'; +import { ExternalHostError } from '../../../types/errors/external-host-error'; import { readLocalFile } from '../../../util/fs'; import * as hostRules from '../../../util/host-rules'; import { flattenPackageRules } from './flatten'; From a905012d11a5340ce55d041dbc2039293d64b397 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Tue, 23 Jun 2020 12:19:25 +0200 Subject: [PATCH 11/25] refactor: getRegistryReleases wrapper --- lib/datasource/index.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/datasource/index.ts b/lib/datasource/index.ts index 63e982127efa6e..1ef183c4e8eec3 100644 --- a/lib/datasource/index.ts +++ b/lib/datasource/index.ts @@ -30,6 +30,15 @@ function load(datasource: string): Promise { type GetReleasesInternalConfig = GetReleasesConfig & GetPkgReleasesConfig; +async function getRegistryReleases( + datasource, + config: GetReleasesConfig, + registryUrl: string +): Promise { + const res = await datasource.getReleases({ ...config, registryUrl }); + return res; +} + function firstRegistry( config: GetReleasesInternalConfig, datasource: Datasource, @@ -42,10 +51,7 @@ function firstRegistry( ); } const registryUrl = registryUrls[0]; - return datasource.getReleases({ - ...config, - registryUrl, - }); + return getRegistryReleases(datasource, config, registryUrl); } async function huntRegistries( @@ -57,10 +63,7 @@ async function huntRegistries( let datasourceError; for (const registryUrl of registryUrls) { try { - res = await datasource.getReleases({ - ...config, - registryUrl, - }); + res = await getRegistryReleases(datasource, config, registryUrl); if (res) { break; } @@ -89,10 +92,7 @@ async function mergeRegistries( let datasourceError; for (const registryUrl of registryUrls) { try { - const res = await datasource.getReleases({ - ...config, - registryUrl, - }); + const res = await getRegistryReleases(datasource, config, registryUrl); if (combinedRes) { combinedRes = { ...res, ...combinedRes }; combinedRes.releases = [...combinedRes.releases, ...res.releases]; From e4e76f8f49770b8d14c6bc289038ba74ff27a6b9 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Tue, 23 Jun 2020 14:40:26 +0200 Subject: [PATCH 12/25] refactor: simplify crate datasource --- .../crate/__fixtures__/invalid_crate_data | 1 - .../crate/__snapshots__/index.spec.ts.snap | 181 ------------------ lib/datasource/crate/index.spec.ts | 22 --- lib/datasource/crate/index.ts | 64 ++----- 4 files changed, 19 insertions(+), 249 deletions(-) delete mode 100644 lib/datasource/crate/__fixtures__/invalid_crate_data diff --git a/lib/datasource/crate/__fixtures__/invalid_crate_data b/lib/datasource/crate/__fixtures__/invalid_crate_data deleted file mode 100644 index 6de95ab075a2ae..00000000000000 --- a/lib/datasource/crate/__fixtures__/invalid_crate_data +++ /dev/null @@ -1 +0,0 @@ -{"name":"some_crate","deps":[{"name":"time","req":"^0.1","features":[],"optional":false,"default_features":true,"target":null,"kind":"normal"},{"name":"threadpool","req":"^0.1.4","features":[],"optional":false,"default_features":true,"target":null,"kind":"normal"}],"cksum":"2884e8cf8d78d9a6de8bbc1898603edb2b625eb8f64e371919906d0fec5660e7","features":{},"yanked":false} \ No newline at end of file diff --git a/lib/datasource/crate/__snapshots__/index.spec.ts.snap b/lib/datasource/crate/__snapshots__/index.spec.ts.snap index f5d2a6e7250c73..b16d093225e8b9 100644 --- a/lib/datasource/crate/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/crate/__snapshots__/index.spec.ts.snap @@ -501,101 +501,6 @@ Array [ ] `; -exports[`datasource/crate getReleases returns null for invalid crate data 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/no/n_/non_existent_crate", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/no/n_/non_existent_crate", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/no/n_/non_existent_crate", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/so/me/some_crate", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/so/me/some_crate", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/so/me/some_crate", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/li/bc/libc", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/am/et/amethyst", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/in/va/invalid-crate-name", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/so/me/some_crate", - }, -] -`; - exports[`datasource/crate getReleases returns null for missing fields 1`] = ` Array [ Object { @@ -678,92 +583,6 @@ Array [ ] `; -exports[`datasource/crate getReleases returns null if crate name is invalid 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/no/n_/non_existent_crate", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/no/n_/non_existent_crate", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/no/n_/non_existent_crate", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/so/me/some_crate", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/so/me/some_crate", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/so/me/some_crate", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/li/bc/libc", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/am/et/amethyst", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/in/va/invalid-crate-name", - }, -] -`; - exports[`datasource/crate getReleases throws for 5xx 1`] = `[Error: external-host-error]`; exports[`datasource/crate getReleases throws for 5xx 2`] = ` diff --git a/lib/datasource/crate/index.spec.ts b/lib/datasource/crate/index.spec.ts index 0821fb55b7b671..5872d9aad32462 100644 --- a/lib/datasource/crate/index.spec.ts +++ b/lib/datasource/crate/index.spec.ts @@ -9,10 +9,6 @@ const res2 = fs.readFileSync( 'lib/datasource/crate/__fixtures__/amethyst', 'utf8' ); -const res3 = fs.readFileSync( - 'lib/datasource/crate/__fixtures__/invalid_crate_data', - 'utf8' -); const baseUrl = 'https://raw.githubusercontent.com/rust-lang/crates.io-index/master/'; @@ -100,23 +96,5 @@ describe('datasource/crate', () => { expect(res).toBeDefined(); expect(httpMock.getTrace()).toMatchSnapshot(); }); - it('returns null if crate name is invalid', async () => { - httpMock.scope(baseUrl).get('/in/va/invalid-crate-name').reply(200, res2); - const res = await getPkgReleases({ - datasource, - depName: 'invalid-crate-name', - }); - expect(res).toBeNull(); - expect(httpMock.getTrace()).toMatchSnapshot(); - }); - it('returns null for invalid crate data', async () => { - httpMock.scope(baseUrl).get('/so/me/some_crate').reply(200, res3); - const res = await getPkgReleases({ - datasource, - depName: 'some_crate', - }); - expect(res).toBeNull(); - expect(httpMock.getTrace()).toMatchSnapshot(); - }); }); }); diff --git a/lib/datasource/crate/index.ts b/lib/datasource/crate/index.ts index 13d2c000d23652..e6185ed3009a5f 100644 --- a/lib/datasource/crate/index.ts +++ b/lib/datasource/crate/index.ts @@ -40,54 +40,28 @@ export async function getReleases({ 'https://raw.githubusercontent.com/rust-lang/crates.io-index/master/'; const crateUrl = baseUrl + path; try { - let res: any = await http.get(crateUrl); - if (!res || !res.body) { - logger.warn( - { dependency: lookupName }, - `Received invalid crate data from ${crateUrl}` - ); - return null; - } - res = res.body; - res = res.split('\n'); - res = res.map((line) => line.trim()).filter((line) => line.length !== 0); - if (res.length === 0) { - logger.warn( - { dependency: lookupName }, - `Received empty list from ${crateUrl}` - ); - return null; - } - // Filter empty lines (takes care of trailing \n) - // eslint-disable-next-line @typescript-eslint/unbound-method - res = res.map(JSON.parse); - if (res[0].name !== lookupName) { - logger.warn( - { dependency: lookupName }, - `Received invalid crate name from ${crateUrl}` - ); - return null; - } - if (!res[0].vers) { - logger.warn( - { dependency: lookupName }, - `Recieved invalid data (vers field doesn't exist) from ${crateUrl}` - ); - return null; - } + const lines = (await http.get(crateUrl)).body + .split('\n') // break into lines + .map((line) => line.trim()) // remove whitespace + .filter((line) => line.length !== 0) // remove empty lines + .map((line) => JSON.parse(line)); // parse const result: ReleaseResult = { releases: [], }; - result.releases = res.map((version: { vers: string; yanked: boolean }) => { - const release: Release = { - version: version.vers, - }; - if (version.yanked) { - release.isDeprecated = true; - } - return release; - }); - + result.releases = lines + .map((version: { vers: string; yanked: boolean }) => { + const release: Release = { + version: version.vers, + }; + if (version.yanked) { + release.isDeprecated = true; + } + return release; + }) + .filter((release) => release.version); + if (!result.releases.length) { + return null; + } const cacheMinutes = 10; await globalCache.set(cacheNamespace, cacheKey, result, cacheMinutes); return result; From 013c63664b0d83fa8d493edc21d612de074566f1 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Wed, 24 Jun 2020 22:33:27 +0200 Subject: [PATCH 13/25] refactor: remove global.renovateCache (#6579) --- lib/globals.d.ts | 18 ------------------ lib/util/cache/global/common.ts | 10 ++++++++++ lib/util/cache/global/file.spec.ts | 23 ++++++++++++----------- lib/util/cache/global/file.ts | 26 +++++++++++++++++--------- lib/util/cache/global/index.spec.ts | 2 -- lib/util/cache/global/index.ts | 19 +++++++++++++++---- lib/util/cache/global/redis.ts | 11 ++++++++--- 7 files changed, 62 insertions(+), 47 deletions(-) create mode 100644 lib/util/cache/global/common.ts diff --git a/lib/globals.d.ts b/lib/globals.d.ts index 79df2b6aedf0cf..2eeb507f25fcec 100644 --- a/lib/globals.d.ts +++ b/lib/globals.d.ts @@ -2,20 +2,6 @@ * This file should be removed in future. */ -declare namespace Renovate { - interface Cache { - get(namespace: string, key: string): Promise; - rm(namespace: string, key: string): Promise; - - set( - namespace: string, - key: string, - value: T, - ttlMinutes?: number - ): Promise; - } -} - declare interface Error { configFile?: string; @@ -30,14 +16,10 @@ declare namespace NodeJS { appMode?: boolean; gitAuthor?: { name: string; email: string }; - renovateCache: Renovate.Cache; - trustLevel?: string; } } -declare let renovateCache: Renovate.Cache; - // can't use `resolveJsonModule` because it will copy json files and change dist path declare module '*.json' { const value: { version: string } & Record; diff --git a/lib/util/cache/global/common.ts b/lib/util/cache/global/common.ts new file mode 100644 index 00000000000000..0742921000bc7c --- /dev/null +++ b/lib/util/cache/global/common.ts @@ -0,0 +1,10 @@ +export interface GlobalCache { + get(namespace: string, key: string): Promise; + + set( + namespace: string, + key: string, + value: T, + ttlMinutes?: number + ): Promise; +} diff --git a/lib/util/cache/global/file.spec.ts b/lib/util/cache/global/file.spec.ts index 26be177a9c8649..d4d26fcac869ce 100644 --- a/lib/util/cache/global/file.spec.ts +++ b/lib/util/cache/global/file.spec.ts @@ -1,24 +1,25 @@ import os from 'os'; -import { init } from './file'; +import { get, init, set } from './file'; describe('lib/util/cache/global/file', () => { - beforeAll(() => { - init(os.tmpdir()); + it('returns if uninitiated', async () => { + await set('test', 'key', 1234); + expect(await get('test', 'key')).toBeUndefined(); }); - it('gets null', async () => { - expect( - await global.renovateCache.get('test', 'missing-key') - ).toBeUndefined(); + init(os.tmpdir()); + expect(await get('test', 'missing-key')).toBeUndefined(); }); it('sets and gets', async () => { - await global.renovateCache.set('test', 'key', 1234); - expect(await global.renovateCache.get('test', 'key')).toBe(1234); + init(os.tmpdir()); + await set('test', 'key', 1234); + expect(await get('test', 'key')).toBe(1234); }); it('expires', async () => { - await global.renovateCache.set('test', 'key', 1234, -5); - expect(await global.renovateCache.get('test', 'key')).toBeUndefined(); + init(os.tmpdir()); + await set('test', 'key', 1234, -5); + expect(await get('test', 'key')).toBeUndefined(); }); }); diff --git a/lib/util/cache/global/file.ts b/lib/util/cache/global/file.ts index 0fcbda3010bd78..e517e612b1320b 100644 --- a/lib/util/cache/global/file.ts +++ b/lib/util/cache/global/file.ts @@ -7,16 +7,22 @@ function getKey(namespace: string, key: string): string { return `${namespace}-${key}`; } -let renovateCache: string; +let cacheFileName: string; async function rm(namespace: string, key: string): Promise { logger.trace({ namespace, key }, 'Removing cache entry'); - await cacache.rm.entry(renovateCache, getKey(namespace, key)); + await cacache.rm.entry(cacheFileName, getKey(namespace, key)); } -async function get(namespace: string, key: string): Promise { +export async function get( + namespace: string, + key: string +): Promise { + if (!cacheFileName) { + return undefined; + } try { - const res = await cacache.get(renovateCache, getKey(namespace, key)); + const res = await cacache.get(cacheFileName, getKey(namespace, key)); const cachedValue = JSON.parse(res.data.toString()); if (cachedValue) { if (DateTime.local() < DateTime.fromISO(cachedValue.expiry)) { @@ -31,15 +37,18 @@ async function get(namespace: string, key: string): Promise { return undefined; } -async function set( +export async function set( namespace: string, key: string, value: unknown, ttlMinutes = 5 ): Promise { + if (!cacheFileName) { + return; + } logger.trace({ namespace, key, ttlMinutes }, 'Saving cached value'); await cacache.put( - renovateCache, + cacheFileName, getKey(namespace, key), JSON.stringify({ value, @@ -49,7 +58,6 @@ async function set( } export function init(cacheDir: string): void { - renovateCache = path.join(cacheDir, '/renovate/renovate-cache-v1'); - logger.debug('Initializing Renovate internal cache into ' + renovateCache); - global.renovateCache = global.renovateCache || { get, set, rm }; + cacheFileName = path.join(cacheDir, '/renovate/renovate-cache-v1'); + logger.debug('Initializing Renovate internal cache into ' + cacheFileName); } diff --git a/lib/util/cache/global/index.spec.ts b/lib/util/cache/global/index.spec.ts index d88d80496a5e35..c91c9d1abeca27 100644 --- a/lib/util/cache/global/index.spec.ts +++ b/lib/util/cache/global/index.spec.ts @@ -10,7 +10,6 @@ describe(getName(__filename), () => { expect(await set('test', 'some-key', 'some-value', 5)).toBeUndefined(); }); it('sets and gets file', async () => { - global.renovateCache = { get: jest.fn(), set: jest.fn(), rm: jest.fn() }; init({ cacheDir: 'some-dir' }); expect( await set('some-namespace', 'some-key', 'some-value', 1) @@ -18,7 +17,6 @@ describe(getName(__filename), () => { expect(await get('some-namespace', 'unknown-key')).toBeUndefined(); }); it('sets and gets redis', async () => { - global.renovateCache = { get: jest.fn(), set: jest.fn(), rm: jest.fn() }; init({ redisUrl: 'some-url' }); expect( await set('some-namespace', 'some-key', 'some-value', 1) diff --git a/lib/util/cache/global/index.ts b/lib/util/cache/global/index.ts index 055f01cae922ab..0fb6a2819aa11f 100644 --- a/lib/util/cache/global/index.ts +++ b/lib/util/cache/global/index.ts @@ -1,19 +1,22 @@ import { RenovateConfig } from '../../../config/common'; import * as runCache from '../run'; +import { GlobalCache } from './common'; import * as fileCache from './file'; import * as redisCache from './redis'; +let cacheProxy: GlobalCache; + function getGlobalKey(namespace: string, key: string): string { return `global%%${namespace}%%${key}`; } export function get(namespace: string, key: string): Promise { - if (!global.renovateCache) { + if (!cacheProxy) { return undefined; } const globalKey = getGlobalKey(namespace, key); if (!runCache.get(globalKey)) { - runCache.set(globalKey, global.renovateCache.get(namespace, key)); + runCache.set(globalKey, cacheProxy.get(namespace, key)); } return runCache.get(globalKey); } @@ -24,19 +27,27 @@ export function set( value: any, minutes: number ): Promise { - if (!global.renovateCache) { + if (!cacheProxy) { return undefined; } const globalKey = getGlobalKey(namespace, key); runCache.set(globalKey, value); - return global.renovateCache.set(namespace, key, value, minutes); + return cacheProxy.set(namespace, key, value, minutes); } export function init(config: RenovateConfig): void { if (config.redisUrl) { redisCache.init(config.redisUrl); + cacheProxy = { + get: redisCache.get, + set: redisCache.set, + }; } else { fileCache.init(config.cacheDir); + cacheProxy = { + get: fileCache.get, + set: fileCache.set, + }; } } diff --git a/lib/util/cache/global/redis.ts b/lib/util/cache/global/redis.ts index bd3406837fe808..b86f7e05b82246 100644 --- a/lib/util/cache/global/redis.ts +++ b/lib/util/cache/global/redis.ts @@ -22,7 +22,13 @@ async function rm(namespace: string, key: string): Promise { await client?.del(getKey(namespace, key)); } -async function get(namespace: string, key: string): Promise { +export async function get( + namespace: string, + key: string +): Promise { + if (!client) { + return undefined; + } logger.trace(`cache.get(${namespace}, ${key})`); try { const res = await client?.get(getKey(namespace, key)); @@ -41,7 +47,7 @@ async function get(namespace: string, key: string): Promise { return undefined; } -async function set( +export async function set( namespace: string, key: string, value: unknown, @@ -73,5 +79,4 @@ export function init(url: string): void { return Math.min(options.attempt * 100, 3000); }, }); - global.renovateCache = { get, set, rm }; } From ceb7c22e042f5185c58867d1daa8f3ea063b4ba4 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Thu, 25 Jun 2020 07:43:15 +0200 Subject: [PATCH 14/25] fix(lerna): ignore scripts (#6581) --- .../npm/post-update/__snapshots__/lerna.spec.ts.snap | 10 +++++----- lib/manager/npm/post-update/lerna.ts | 5 ++++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/manager/npm/post-update/__snapshots__/lerna.spec.ts.snap b/lib/manager/npm/post-update/__snapshots__/lerna.spec.ts.snap index 3bf5608e95b52a..1a1bed7e54f8ef 100644 --- a/lib/manager/npm/post-update/__snapshots__/lerna.spec.ts.snap +++ b/lib/manager/npm/post-update/__snapshots__/lerna.spec.ts.snap @@ -59,7 +59,7 @@ Array [ }, }, Object { - "cmd": "lerna bootstrap --no-ci -- --ignore-scripts --no-audit --package-lock-only", + "cmd": "lerna bootstrap --no-ci --ignore-scripts -- --ignore-scripts --no-audit --package-lock-only", "options": Object { "cwd": "some-dir", "encoding": "utf-8", @@ -98,7 +98,7 @@ Array [ }, }, Object { - "cmd": "lerna bootstrap --no-ci -- --ignore-scripts --no-audit --package-lock-only", + "cmd": "lerna bootstrap --no-ci --ignore-scripts -- --ignore-scripts --no-audit --package-lock-only", "options": Object { "cwd": "some-dir", "encoding": "utf-8", @@ -137,7 +137,7 @@ Array [ }, }, Object { - "cmd": "lerna bootstrap --no-ci -- --ignore-scripts --ignore-engines --ignore-platform", + "cmd": "lerna bootstrap --no-ci --ignore-scripts -- --ignore-scripts --ignore-engines --ignore-platform", "options": Object { "cwd": "some-dir", "encoding": "utf-8", @@ -176,7 +176,7 @@ Array [ }, }, Object { - "cmd": "lerna bootstrap --no-ci -- --ignore-scripts --no-audit --package-lock-only", + "cmd": "lerna bootstrap --no-ci --ignore-scripts -- --ignore-scripts --no-audit --package-lock-only", "options": Object { "cwd": "some-dir", "encoding": "utf-8", @@ -215,7 +215,7 @@ Array [ }, }, Object { - "cmd": "lerna bootstrap --no-ci -- --ignore-scripts --no-audit", + "cmd": "lerna bootstrap --no-ci --ignore-scripts -- --ignore-scripts --no-audit", "options": Object { "cwd": "some-dir", "encoding": "utf-8", diff --git a/lib/manager/npm/post-update/lerna.ts b/lib/manager/npm/post-update/lerna.ts index 41e79c18a6ce39..2a7607ecf96b17 100644 --- a/lib/manager/npm/post-update/lerna.ts +++ b/lib/manager/npm/post-update/lerna.ts @@ -55,9 +55,12 @@ export async function generateLockFiles( logger.warn({ lernaClient }, 'Unknown lernaClient'); return { error: false }; } + let lernaCommand = `lerna bootstrap --no-ci --ignore-scripts -- `; if (global.trustLevel === 'high' && config.ignoreScripts !== false) { cmdOptions = cmdOptions.replace('--ignore-scripts ', ''); + lernaCommand = lernaCommand.replace('--ignore-scripts ', ''); } + lernaCommand += cmdOptions; const tagConstraint = await getNodeConstraint(config); const execOptions: ExecOptions = { cwd, @@ -93,7 +96,7 @@ export async function generateLockFiles( } logger.debug('Using lerna version ' + lernaVersion); preCommands.push(`npm i -g lerna@${quote(lernaVersion)}`); - cmd.push(`lerna bootstrap --no-ci -- ${cmdOptions}`); + cmd.push(lernaCommand); await exec(cmd, execOptions); } catch (err) /* istanbul ignore next */ { logger.debug( From be37109cf697103eb948893e2e3874c6cedc9fbd Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Thu, 25 Jun 2020 08:32:55 +0200 Subject: [PATCH 15/25] refactor: rename globalCache to packageCache (#6580) --- lib/datasource/cache.ts | 9 ++++++--- lib/datasource/crate/index.ts | 6 +++--- lib/datasource/docker/index.ts | 14 +++++++------- lib/datasource/galaxy/index.ts | 6 +++--- lib/datasource/git-refs/index.ts | 6 +++--- lib/datasource/git-submodules/index.ts | 6 +++--- lib/datasource/github-releases/index.ts | 6 +++--- lib/datasource/github-tags/index.ts | 14 +++++++------- lib/datasource/gitlab-tags/index.ts | 6 +++--- lib/datasource/helm/index.ts | 6 +++--- lib/datasource/maven/index.ts | 10 +++++----- lib/datasource/npm/get.ts | 6 +++--- lib/datasource/nuget/v3.ts | 6 +++--- lib/datasource/orb/index.ts | 6 +++--- lib/datasource/packagist/index.ts | 10 +++++----- lib/datasource/pod/index.ts | 6 +++--- lib/datasource/repology/index.ts | 8 ++++---- lib/datasource/ruby-version/index.ts | 6 +++--- lib/datasource/terraform-module/index.ts | 6 +++--- lib/manager/bazel/update.ts | 6 +++--- lib/util/cache/{global => package}/common.ts | 2 +- lib/util/cache/{global => package}/file.spec.ts | 0 lib/util/cache/{global => package}/file.ts | 0 lib/util/cache/{global => package}/index.spec.ts | 0 lib/util/cache/{global => package}/index.ts | 4 ++-- lib/util/cache/{global => package}/redis.ts | 0 lib/workers/global/initialize.ts | 6 +++--- lib/workers/pr/changelog/release-notes.ts | 6 +++--- lib/workers/pr/changelog/source-github.ts | 6 +++--- lib/workers/pr/changelog/source-gitlab.ts | 6 +++--- 30 files changed, 91 insertions(+), 88 deletions(-) rename lib/util/cache/{global => package}/common.ts (84%) rename lib/util/cache/{global => package}/file.spec.ts (100%) rename lib/util/cache/{global => package}/file.ts (100%) rename lib/util/cache/{global => package}/index.spec.ts (100%) rename lib/util/cache/{global => package}/index.ts (94%) rename lib/util/cache/{global => package}/redis.ts (100%) diff --git a/lib/datasource/cache.ts b/lib/datasource/cache.ts index ecd74cbe79e176..bab0cf1dbf81bb 100644 --- a/lib/datasource/cache.ts +++ b/lib/datasource/cache.ts @@ -1,5 +1,5 @@ import { logger } from '../logger'; -import * as globalCache from '../util/cache/global'; +import * as packageCache from '../util/cache/package'; /** * Cache callback result which has to be returned by the `CacheCallback` function. @@ -58,7 +58,10 @@ export async function cacheAble({ }: CacheConfig): Promise { const cacheNamespace = `datasource-${id}`; const cacheKey = JSON.stringify(lookup); - const cachedResult = await globalCache.get(cacheNamespace, cacheKey); + const cachedResult = await packageCache.get( + cacheNamespace, + cacheKey + ); // istanbul ignore if if (cachedResult) { logger.trace({ id, lookup }, 'datasource cachedResult'); @@ -69,7 +72,7 @@ export async function cacheAble({ if (isPrivate) { logger.trace({ id, lookup }, 'Skipping datasource cache for private data'); } else { - await globalCache.set(cacheNamespace, cacheKey, data, minutes); + await packageCache.set(cacheNamespace, cacheKey, data, minutes); } return data; } diff --git a/lib/datasource/crate/index.ts b/lib/datasource/crate/index.ts index e6185ed3009a5f..9715b4eab18353 100644 --- a/lib/datasource/crate/index.ts +++ b/lib/datasource/crate/index.ts @@ -1,6 +1,6 @@ import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; -import * as globalCache from '../../util/cache/global'; +import * as packageCache from '../../util/cache/package'; import { Http } from '../../util/http'; import { GetReleasesConfig, Release, ReleaseResult } from '../common'; @@ -13,7 +13,7 @@ export async function getReleases({ }: GetReleasesConfig): Promise { const cacheNamespace = 'datasource-crate'; const cacheKey = lookupName; - const cachedResult = await globalCache.get( + const cachedResult = await packageCache.get( cacheNamespace, cacheKey ); @@ -63,7 +63,7 @@ export async function getReleases({ return null; } const cacheMinutes = 10; - await globalCache.set(cacheNamespace, cacheKey, result, cacheMinutes); + await packageCache.set(cacheNamespace, cacheKey, result, cacheMinutes); return result; } catch (err) { if (err.statusCode === 404 || err.code === 'ENOTFOUND') { diff --git a/lib/datasource/docker/index.ts b/lib/datasource/docker/index.ts index a295738843c9e3..4af0da52eac34e 100644 --- a/lib/datasource/docker/index.ts +++ b/lib/datasource/docker/index.ts @@ -7,7 +7,7 @@ import wwwAuthenticate from 'www-authenticate'; import { logger } from '../../logger'; import { HostRule } from '../../types'; import { ExternalHostError } from '../../types/errors/external-host-error'; -import * as globalCache from '../../util/cache/global'; +import * as packageCache from '../../util/cache/package'; import * as hostRules from '../../util/host-rules'; import { Http, HttpResponse } from '../../util/http'; import { GetReleasesConfig, ReleaseResult } from '../common'; @@ -342,7 +342,7 @@ export async function getDigest( const cacheKey = `${registry}:${repository}:${newTag}`; let digest = null; try { - const cachedResult = await globalCache.get(cacheNamespace, cacheKey); + const cachedResult = await packageCache.get(cacheNamespace, cacheKey); // istanbul ignore if if (cachedResult !== undefined) { return cachedResult; @@ -370,7 +370,7 @@ export async function getDigest( ); } const cacheMinutes = 30; - await globalCache.set(cacheNamespace, cacheKey, digest, cacheMinutes); + await packageCache.set(cacheNamespace, cacheKey, digest, cacheMinutes); return digest; } @@ -382,7 +382,7 @@ async function getTags( try { const cacheNamespace = 'datasource-docker-tags'; const cacheKey = `${registry}:${repository}`; - const cachedResult = await globalCache.get( + const cachedResult = await packageCache.get( cacheNamespace, cacheKey ); @@ -411,7 +411,7 @@ async function getTags( page += 1; } while (url && page < 20); const cacheMinutes = 15; - await globalCache.set(cacheNamespace, cacheKey, tags, cacheMinutes); + await packageCache.set(cacheNamespace, cacheKey, tags, cacheMinutes); return tags; } catch (err) /* istanbul ignore next */ { if (err instanceof ExternalHostError) { @@ -482,7 +482,7 @@ async function getLabels( logger.debug(`getLabels(${registry}, ${repository}, ${tag})`); const cacheNamespace = 'datasource-docker-labels'; const cacheKey = `${registry}:${repository}:${tag}`; - const cachedResult = await globalCache.get>( + const cachedResult = await packageCache.get>( cacheNamespace, cacheKey ); @@ -541,7 +541,7 @@ async function getLabels( ); } const cacheMinutes = 60; - await globalCache.set(cacheNamespace, cacheKey, labels, cacheMinutes); + await packageCache.set(cacheNamespace, cacheKey, labels, cacheMinutes); return labels; } catch (err) { if (err instanceof ExternalHostError) { diff --git a/lib/datasource/galaxy/index.ts b/lib/datasource/galaxy/index.ts index 14c67e18832feb..ed01e05fe8a442 100644 --- a/lib/datasource/galaxy/index.ts +++ b/lib/datasource/galaxy/index.ts @@ -1,6 +1,6 @@ import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; -import * as globalCache from '../../util/cache/global'; +import * as packageCache from '../../util/cache/package'; import { Http } from '../../util/http'; import { GetReleasesConfig, Release, ReleaseResult } from '../common'; @@ -13,7 +13,7 @@ export async function getReleases({ }: GetReleasesConfig): Promise { const cacheNamespace = 'datasource-galaxy'; const cacheKey = lookupName; - const cachedResult = await globalCache.get( + const cachedResult = await packageCache.get( cacheNamespace, cacheKey ); @@ -90,7 +90,7 @@ export async function getReleases({ } ); const cacheMinutes = 10; - await globalCache.set(cacheNamespace, cacheKey, result, cacheMinutes); + await packageCache.set(cacheNamespace, cacheKey, result, cacheMinutes); return result; } catch (err) { if ( diff --git a/lib/datasource/git-refs/index.ts b/lib/datasource/git-refs/index.ts index afd75aed8c0bd0..50820ee5920950 100644 --- a/lib/datasource/git-refs/index.ts +++ b/lib/datasource/git-refs/index.ts @@ -1,6 +1,6 @@ import simpleGit from 'simple-git/promise'; import { logger } from '../../logger'; -import * as globalCache from '../../util/cache/global'; +import * as packageCache from '../../util/cache/package'; import * as semver from '../../versioning/semver'; import { DigestConfig, GetReleasesConfig, ReleaseResult } from '../common'; @@ -24,7 +24,7 @@ export async function getRawRefs({ try { const cacheNamespace = 'git-raw-refs'; - const cachedResult = await globalCache.get( + const cachedResult = await packageCache.get( cacheNamespace, lookupName ); @@ -68,7 +68,7 @@ export async function getRawRefs({ }) .filter(Boolean) .filter((ref) => ref.type !== 'pull' && !ref.value.endsWith('^{}')); - await globalCache.set(cacheNamespace, lookupName, refs, cacheMinutes); + await packageCache.set(cacheNamespace, lookupName, refs, cacheMinutes); return refs; } catch (err) { logger.info({ err }, `Git-Raw-Refs lookup error in ${lookupName}`); diff --git a/lib/datasource/git-submodules/index.ts b/lib/datasource/git-submodules/index.ts index b00fe9afa52dd4..4a812a3fb748f9 100644 --- a/lib/datasource/git-submodules/index.ts +++ b/lib/datasource/git-submodules/index.ts @@ -2,7 +2,7 @@ import { URL } from 'url'; import Git from 'simple-git/promise'; import { logger } from '../../logger'; -import * as globalCache from '../../util/cache/global'; +import * as packageCache from '../../util/cache/package'; import { DigestConfig, GetReleasesConfig, ReleaseResult } from '../common'; export const id = 'git-submodules'; @@ -13,7 +13,7 @@ export async function getReleases({ }: GetReleasesConfig): Promise { const cacheNamespace = 'datasource-git-submodules'; const cacheKey = `${registryUrls[0]}-${registryUrls[1]}`; - const cachedResult = await globalCache.get( + const cachedResult = await packageCache.get( cacheNamespace, cacheKey ); @@ -42,7 +42,7 @@ export async function getReleases({ ], }; const cacheMinutes = 60; - await globalCache.set(cacheNamespace, cacheKey, result, cacheMinutes); + await packageCache.set(cacheNamespace, cacheKey, result, cacheMinutes); return result; } catch (err) { logger.debug({ err }, `Git-SubModules lookup error in ${lookupName}`); diff --git a/lib/datasource/github-releases/index.ts b/lib/datasource/github-releases/index.ts index 8b4c2c89dcd503..438f6509ac5f6a 100644 --- a/lib/datasource/github-releases/index.ts +++ b/lib/datasource/github-releases/index.ts @@ -1,5 +1,5 @@ import { logger } from '../../logger'; -import * as globalCache from '../../util/cache/global'; +import * as packageCache from '../../util/cache/package'; import { GithubHttp } from '../../util/http/github'; import { GetReleasesConfig, ReleaseResult } from '../common'; @@ -28,7 +28,7 @@ export async function getReleases({ lookupName: repo, }: GetReleasesConfig): Promise { let githubReleases: GithubRelease[]; - const cachedResult = await globalCache.get( + const cachedResult = await packageCache.get( cacheNamespace, repo ); @@ -59,6 +59,6 @@ export async function getReleases({ releaseTimestamp: published_at, })); const cacheMinutes = 10; - await globalCache.set(cacheNamespace, repo, dependency, cacheMinutes); + await packageCache.set(cacheNamespace, repo, dependency, cacheMinutes); return dependency; } diff --git a/lib/datasource/github-tags/index.ts b/lib/datasource/github-tags/index.ts index fea2bae705bd85..d2a385a42bafac 100644 --- a/lib/datasource/github-tags/index.ts +++ b/lib/datasource/github-tags/index.ts @@ -1,5 +1,5 @@ import { logger } from '../../logger'; -import * as globalCache from '../../util/cache/global'; +import * as packageCache from '../../util/cache/package'; import { GithubHttp } from '../../util/http/github'; import { DigestConfig, GetReleasesConfig, ReleaseResult } from '../common'; @@ -24,7 +24,7 @@ async function getTagCommit( githubRepo: string, tag: string ): Promise { - const cachedResult = await globalCache.get( + const cachedResult = await packageCache.get( cacheNamespace, getCacheKey(githubRepo, `tag-${tag}`) ); @@ -53,7 +53,7 @@ async function getTagCommit( return null; } const cacheMinutes = 120; - await globalCache.set( + await packageCache.set( cacheNamespace, getCacheKey(githubRepo, `tag-${tag}`), digest, @@ -76,7 +76,7 @@ export async function getDigest( if (newValue && newValue.length) { return getTagCommit(githubRepo, newValue); } - const cachedResult = await globalCache.get( + const cachedResult = await packageCache.get( cacheNamespace, getCacheKey(githubRepo, 'commit') ); @@ -99,7 +99,7 @@ export async function getDigest( return null; } const cacheMinutes = 10; - await globalCache.set( + await packageCache.set( cacheNamespace, getCacheKey(githubRepo, 'commit'), digest, @@ -122,7 +122,7 @@ export async function getReleases({ lookupName: repo, }: GetReleasesConfig): Promise { let versions: string[]; - const cachedResult = await globalCache.get( + const cachedResult = await packageCache.get( cacheNamespace, getCacheKey(repo, 'tags') ); @@ -157,7 +157,7 @@ export async function getReleases({ gitRef: version, })); const cacheMinutes = 10; - await globalCache.set( + await packageCache.set( cacheNamespace, getCacheKey(repo, 'tags'), dependency, diff --git a/lib/datasource/gitlab-tags/index.ts b/lib/datasource/gitlab-tags/index.ts index f9c795ce24ed6b..59bff4283b83e6 100644 --- a/lib/datasource/gitlab-tags/index.ts +++ b/lib/datasource/gitlab-tags/index.ts @@ -1,6 +1,6 @@ import URL from 'url'; import { logger } from '../../logger'; -import * as globalCache from '../../util/cache/global'; +import * as packageCache from '../../util/cache/package'; import { GitlabHttp } from '../../util/http/gitlab'; import { GetReleasesConfig, ReleaseResult } from '../common'; @@ -28,7 +28,7 @@ export async function getReleases({ lookupName: repo, }: GetReleasesConfig): Promise { let gitlabTags: GitlabTag[]; - const cachedResult = await globalCache.get( + const cachedResult = await packageCache.get( cacheNamespace, getCacheKey(depHost, repo) ); @@ -72,7 +72,7 @@ export async function getReleases({ })); const cacheMinutes = 10; - await globalCache.set( + await packageCache.set( cacheNamespace, getCacheKey(depHost, repo), dependency, diff --git a/lib/datasource/helm/index.ts b/lib/datasource/helm/index.ts index 24a1910d78937b..def7776dbc4f19 100644 --- a/lib/datasource/helm/index.ts +++ b/lib/datasource/helm/index.ts @@ -2,7 +2,7 @@ import yaml from 'js-yaml'; import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; -import * as globalCache from '../../util/cache/global'; +import * as packageCache from '../../util/cache/package'; import { Http } from '../../util/http'; import { ensureTrailingSlash } from '../../util/url'; import { GetReleasesConfig, ReleaseResult } from '../common'; @@ -21,7 +21,7 @@ export async function getRepositoryData( ): Promise { const cacheNamespace = 'datasource-helm'; const cacheKey = repository; - const cachedIndex = await globalCache.get(cacheNamespace, cacheKey); + const cachedIndex = await packageCache.get(cacheNamespace, cacheKey); // istanbul ignore if if (cachedIndex) { return cachedIndex; @@ -92,7 +92,7 @@ export async function getRepositoryData( }) ); const cacheMinutes = 20; - await globalCache.set(cacheNamespace, cacheKey, result, cacheMinutes); + await packageCache.set(cacheNamespace, cacheKey, result, cacheMinutes); return result; } catch (err) { logger.warn(`Failed to parse index.yaml from ${repository}`); diff --git a/lib/datasource/maven/index.ts b/lib/datasource/maven/index.ts index 8c1337fd1b2355..636a9ea35db7da 100644 --- a/lib/datasource/maven/index.ts +++ b/lib/datasource/maven/index.ts @@ -3,7 +3,7 @@ import fs from 'fs-extra'; import pAll from 'p-all'; import { XmlDocument } from 'xmldoc'; import { logger } from '../../logger'; -import * as globalCache from '../../util/cache/global'; +import * as packageCache from '../../util/cache/package'; import mavenVersion from '../../versioning/maven'; import { compare } from '../../versioning/maven/compare'; import { GetReleasesConfig, ReleaseResult } from '../common'; @@ -158,7 +158,7 @@ async function getVersionsFromMetadata( const cacheNamespace = 'datasource-maven-metadata'; const cacheKey = metadataUrl.toString(); - const cachedVersions = await globalCache.get( + const cachedVersions = await packageCache.get( cacheNamespace, cacheKey ); @@ -173,7 +173,7 @@ async function getVersionsFromMetadata( } const versions = extractVersions(mavenMetadata); - await globalCache.set(cacheNamespace, cacheKey, versions, 10); + await packageCache.set(cacheNamespace, cacheKey, versions, 10); return versions; } @@ -211,7 +211,7 @@ async function filterMissingArtifacts( ): Promise { const cacheNamespace = 'datasource-maven-metadata'; const cacheKey = dependency.dependencyUrl; - let artifactsInfo: ArtifactsInfo | null = await globalCache.get< + let artifactsInfo: ArtifactsInfo | null = await packageCache.get< ArtifactsInfo >(cacheNamespace, cacheKey); @@ -243,7 +243,7 @@ async function filterMissingArtifacts( ? 60 : 24 * 60; - await globalCache.set(cacheNamespace, cacheKey, artifactsInfo, cacheTTL); + await packageCache.set(cacheNamespace, cacheKey, artifactsInfo, cacheTTL); } return versions.filter((v) => artifactsInfo[v]); diff --git a/lib/datasource/npm/get.ts b/lib/datasource/npm/get.ts index 2dc1a9047dec7e..932d21f02b455f 100644 --- a/lib/datasource/npm/get.ts +++ b/lib/datasource/npm/get.ts @@ -7,7 +7,7 @@ import registryAuthToken from 'registry-auth-token'; import getRegistryUrl from 'registry-auth-token/registry-url'; import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; -import * as globalCache from '../../util/cache/global'; +import * as packageCache from '../../util/cache/package'; import { find } from '../../util/host-rules'; import { Http, HttpOptions } from '../../util/http'; import { maskToken } from '../../util/mask'; @@ -71,7 +71,7 @@ export async function getDependency( ); // Now check the persistent cache const cacheNamespace = 'datasource-npm'; - const cachedResult = await globalCache.get( + const cachedResult = await packageCache.get( cacheNamespace, pkgUrl ); @@ -220,7 +220,7 @@ export async function getDependency( whitelistedPublicScopes.includes(scope) || !packageName.startsWith('@') ) { - await globalCache.set(cacheNamespace, pkgUrl, dep, cacheMinutes); + await packageCache.set(cacheNamespace, pkgUrl, dep, cacheMinutes); } return dep; } catch (err) { diff --git a/lib/datasource/nuget/v3.ts b/lib/datasource/nuget/v3.ts index 507af6cbea3542..58f164e584ca7c 100644 --- a/lib/datasource/nuget/v3.ts +++ b/lib/datasource/nuget/v3.ts @@ -1,7 +1,7 @@ import * as semver from 'semver'; import { XmlDocument } from 'xmldoc'; import { logger } from '../../logger'; -import * as globalCache from '../../util/cache/global'; +import * as packageCache from '../../util/cache/package'; import { Http } from '../../util/http'; import { ReleaseResult } from '../common'; @@ -21,7 +21,7 @@ export async function getQueryUrl(url: string): Promise { // https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource const resourceType = 'SearchQueryService'; const cacheKey = `${url}:${resourceType}`; - const cachedResult = await globalCache.get(cacheNamespace, cacheKey); + const cachedResult = await packageCache.get(cacheNamespace, cacheKey); // istanbul ignore if if (cachedResult) { @@ -38,7 +38,7 @@ export async function getQueryUrl(url: string): Promise { const searchQueryServiceId = searchQueryService['@id']; const cacheMinutes = 60; - await globalCache.set( + await packageCache.set( cacheNamespace, cacheKey, searchQueryServiceId, diff --git a/lib/datasource/orb/index.ts b/lib/datasource/orb/index.ts index 0d54db74e9217f..b26c468dc45dcd 100644 --- a/lib/datasource/orb/index.ts +++ b/lib/datasource/orb/index.ts @@ -1,5 +1,5 @@ import { logger } from '../../logger'; -import * as globalCache from '../../util/cache/global'; +import * as packageCache from '../../util/cache/package'; import { Http } from '../../util/http'; import { GetReleasesConfig, ReleaseResult } from '../common'; @@ -26,7 +26,7 @@ export async function getReleases({ logger.debug({ lookupName }, 'orb.getReleases()'); const cacheNamespace = 'orb'; const cacheKey = lookupName; - const cachedResult = await globalCache.get( + const cachedResult = await packageCache.get( cacheNamespace, cacheKey ); @@ -66,7 +66,7 @@ export async function getReleases({ })); logger.trace({ dep }, 'dep'); const cacheMinutes = 15; - await globalCache.set(cacheNamespace, cacheKey, dep, cacheMinutes); + await packageCache.set(cacheNamespace, cacheKey, dep, cacheMinutes); return dep; } catch (err) /* istanbul ignore next */ { logger.debug({ err }, 'CircleCI Orb lookup error'); diff --git a/lib/datasource/packagist/index.ts b/lib/datasource/packagist/index.ts index e0cdfc57ec694c..e75d2602f7a66f 100644 --- a/lib/datasource/packagist/index.ts +++ b/lib/datasource/packagist/index.ts @@ -3,7 +3,7 @@ import URL from 'url'; import pAll from 'p-all'; import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; -import * as globalCache from '../../util/cache/global'; +import * as packageCache from '../../util/cache/package'; import * as runCache from '../../util/cache/run'; import * as hostRules from '../../util/host-rules'; import { Http, HttpOptions } from '../../util/http'; @@ -127,7 +127,7 @@ async function getPackagistFile( const cacheNamespace = 'datasource-packagist-files'; const cacheKey = regUrl + key; // Check the persistent cache for public registries - const cachedResult = await globalCache.get(cacheNamespace, cacheKey); + const cachedResult = await packageCache.get(cacheNamespace, cacheKey); // istanbul ignore if if (cachedResult && cachedResult.sha256 === sha256) { return cachedResult.res; @@ -135,7 +135,7 @@ async function getPackagistFile( const res = (await http.getJson(regUrl + '/' + fileName, opts)) .body; const cacheMinutes = 1440; // 1 day - await globalCache.set( + await packageCache.set( cacheNamespace, cacheKey, { res, sha256 }, @@ -233,7 +233,7 @@ function getAllCachedPackages(regUrl: string): Promise { async function packagistOrgLookup(name: string): Promise { const cacheNamespace = 'datasource-packagist-org'; - const cachedResult = await globalCache.get( + const cachedResult = await packageCache.get( cacheNamespace, name ); @@ -252,7 +252,7 @@ async function packagistOrgLookup(name: string): Promise { logger.trace({ dep }, 'dep'); } const cacheMinutes = 10; - await globalCache.set(cacheNamespace, name, dep, cacheMinutes); + await packageCache.set(cacheNamespace, name, dep, cacheMinutes); return dep; } diff --git a/lib/datasource/pod/index.ts b/lib/datasource/pod/index.ts index 00b6853cabbc28..6832f323ca12dc 100644 --- a/lib/datasource/pod/index.ts +++ b/lib/datasource/pod/index.ts @@ -1,7 +1,7 @@ import crypto from 'crypto'; import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; -import * as globalCache from '../../util/cache/global'; +import * as packageCache from '../../util/cache/package'; import { Http } from '../../util/http'; import { GithubHttp } from '../../util/http/github'; import { GetReleasesConfig, ReleaseResult } from '../common'; @@ -155,7 +155,7 @@ export async function getReleases({ }: GetReleasesConfig): Promise { const podName = lookupName.replace(/\/.*$/, ''); - const cachedResult = await globalCache.get( + const cachedResult = await packageCache.get( cacheNamespace, registryUrl + podName ); @@ -180,7 +180,7 @@ export async function getReleases({ result = await getReleasesFromCDN(podName, baseUrl); } - await globalCache.set(cacheNamespace, podName, result, cacheMinutes); + await packageCache.set(cacheNamespace, podName, result, cacheMinutes); return result; } diff --git a/lib/datasource/repology/index.ts b/lib/datasource/repology/index.ts index 784b72952e3c07..c1617de6cad53e 100644 --- a/lib/datasource/repology/index.ts +++ b/lib/datasource/repology/index.ts @@ -1,7 +1,7 @@ import { URLSearchParams } from 'url'; import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; -import * as globalCache from '../../util/cache/global'; +import * as packageCache from '../../util/cache/package'; import { Http } from '../../util/http'; import { GetReleasesConfig, ReleaseResult } from '../common'; @@ -85,7 +85,7 @@ async function getCachedPackage( ): Promise { // Fetch previous result from cache if available const cacheKey = `${repoName}/${pkgName}`; - const cachedResult = await globalCache.get( + const cachedResult = await packageCache.get( cacheNamespace, cacheKey ); @@ -97,14 +97,14 @@ async function getCachedPackage( // Attempt a binary package lookup and return if successfully const binPkg = await queryPackage(repoName, pkgName, 'binname'); if (binPkg) { - await globalCache.set(cacheNamespace, cacheKey, binPkg, cacheMinutes); + await packageCache.set(cacheNamespace, cacheKey, binPkg, cacheMinutes); return binPkg; } // Otherwise, attempt a source package lookup and return if successfully const srcPkg = await queryPackage(repoName, pkgName, 'srcname'); if (srcPkg) { - await globalCache.set(cacheNamespace, cacheKey, srcPkg, cacheMinutes); + await packageCache.set(cacheNamespace, cacheKey, srcPkg, cacheMinutes); return srcPkg; } diff --git a/lib/datasource/ruby-version/index.ts b/lib/datasource/ruby-version/index.ts index 433ede5c0612cd..fa43d545c5e17e 100644 --- a/lib/datasource/ruby-version/index.ts +++ b/lib/datasource/ruby-version/index.ts @@ -1,7 +1,7 @@ import { parse } from 'node-html-parser'; import { ExternalHostError } from '../../types/errors/external-host-error'; -import * as globalCache from '../../util/cache/global'; +import * as packageCache from '../../util/cache/package'; import { Http } from '../../util/http'; import { isVersion } from '../../versioning/ruby'; import { GetReleasesConfig, ReleaseResult } from '../common'; @@ -17,7 +17,7 @@ export async function getReleases( ): Promise { // First check the persistent cache const cacheNamespace = 'datasource-ruby-version'; - const cachedResult = await globalCache.get( + const cachedResult = await packageCache.get( cacheNamespace, 'all' ); @@ -49,7 +49,7 @@ export async function getReleases( } } } - await globalCache.set(cacheNamespace, 'all', res, 15); + await packageCache.set(cacheNamespace, 'all', res, 15); return res; } catch (err) { throw new ExternalHostError(err); diff --git a/lib/datasource/terraform-module/index.ts b/lib/datasource/terraform-module/index.ts index 5747efb1110730..217cdcb9827294 100644 --- a/lib/datasource/terraform-module/index.ts +++ b/lib/datasource/terraform-module/index.ts @@ -1,6 +1,6 @@ import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; -import * as globalCache from '../../util/cache/global'; +import * as packageCache from '../../util/cache/package'; import { Http } from '../../util/http'; import { GetReleasesConfig, ReleaseResult } from '../common'; @@ -66,7 +66,7 @@ export async function getReleases({ ); const cacheNamespace = 'terraform-module'; const pkgUrl = `${registry}/v1/modules/${repository}`; - const cachedResult = await globalCache.get( + const cachedResult = await packageCache.get( cacheNamespace, pkgUrl ); @@ -98,7 +98,7 @@ export async function getReleases({ } logger.trace({ dep }, 'dep'); const cacheMinutes = 30; - await globalCache.set(cacheNamespace, pkgUrl, dep, cacheMinutes); + await packageCache.set(cacheNamespace, pkgUrl, dep, cacheMinutes); return dep; } catch (err) { if (err.statusCode === 404 || err.code === 'ENOTFOUND') { diff --git a/lib/manager/bazel/update.ts b/lib/manager/bazel/update.ts index 9d9def06e3e578..8f3110cbec4435 100644 --- a/lib/manager/bazel/update.ts +++ b/lib/manager/bazel/update.ts @@ -1,6 +1,6 @@ import { fromStream } from 'hasha'; import { logger } from '../../logger'; -import * as globalCache from '../../util/cache/global'; +import * as packageCache from '../../util/cache/package'; import { Http } from '../../util/http'; import { regEx } from '../../util/regex'; import { UpdateDependencyConfig } from '../common'; @@ -46,7 +46,7 @@ function extractUrls(content: string): string[] | null { async function getHashFromUrl(url: string): Promise { const cacheNamespace = 'url-sha256'; - const cachedResult = await globalCache.get( + const cachedResult = await packageCache.get( cacheNamespace, url ); @@ -59,7 +59,7 @@ async function getHashFromUrl(url: string): Promise { algorithm: 'sha256', }); const cacheMinutes = 3 * 24 * 60; // 3 days - await globalCache.set(cacheNamespace, url, hash, cacheMinutes); + await packageCache.set(cacheNamespace, url, hash, cacheMinutes); return hash; } catch (err) /* istanbul ignore next */ { return null; diff --git a/lib/util/cache/global/common.ts b/lib/util/cache/package/common.ts similarity index 84% rename from lib/util/cache/global/common.ts rename to lib/util/cache/package/common.ts index 0742921000bc7c..89896e47ac5135 100644 --- a/lib/util/cache/global/common.ts +++ b/lib/util/cache/package/common.ts @@ -1,4 +1,4 @@ -export interface GlobalCache { +export interface PackageCache { get(namespace: string, key: string): Promise; set( diff --git a/lib/util/cache/global/file.spec.ts b/lib/util/cache/package/file.spec.ts similarity index 100% rename from lib/util/cache/global/file.spec.ts rename to lib/util/cache/package/file.spec.ts diff --git a/lib/util/cache/global/file.ts b/lib/util/cache/package/file.ts similarity index 100% rename from lib/util/cache/global/file.ts rename to lib/util/cache/package/file.ts diff --git a/lib/util/cache/global/index.spec.ts b/lib/util/cache/package/index.spec.ts similarity index 100% rename from lib/util/cache/global/index.spec.ts rename to lib/util/cache/package/index.spec.ts diff --git a/lib/util/cache/global/index.ts b/lib/util/cache/package/index.ts similarity index 94% rename from lib/util/cache/global/index.ts rename to lib/util/cache/package/index.ts index 0fb6a2819aa11f..d77fdcd8be412e 100644 --- a/lib/util/cache/global/index.ts +++ b/lib/util/cache/package/index.ts @@ -1,10 +1,10 @@ import { RenovateConfig } from '../../../config/common'; import * as runCache from '../run'; -import { GlobalCache } from './common'; +import { PackageCache } from './common'; import * as fileCache from './file'; import * as redisCache from './redis'; -let cacheProxy: GlobalCache; +let cacheProxy: PackageCache; function getGlobalKey(namespace: string, key: string): string { return `global%%${namespace}%%${key}`; diff --git a/lib/util/cache/global/redis.ts b/lib/util/cache/package/redis.ts similarity index 100% rename from lib/util/cache/global/redis.ts rename to lib/util/cache/package/redis.ts diff --git a/lib/workers/global/initialize.ts b/lib/workers/global/initialize.ts index 4e36e1f54c45c4..f575084b75e446 100644 --- a/lib/workers/global/initialize.ts +++ b/lib/workers/global/initialize.ts @@ -4,7 +4,7 @@ import fs from 'fs-extra'; import { RenovateConfig } from '../../config/common'; import { logger } from '../../logger'; import { initPlatform } from '../../platform'; -import * as globalCache from '../../util/cache/global'; +import * as packageCache from '../../util/cache/package'; import { setEmojiConfig } from '../../util/emoji'; import * as limits from './limits'; @@ -34,12 +34,12 @@ export async function globalInitialize( let config = config_; config = await initPlatform(config); config = await setDirectories(config); - globalCache.init(config); + packageCache.init(config); limits.init(config); setEmojiConfig(config); return config; } export function globalFinalize(config: RenovateConfig): void { - globalCache.cleanup(config); + packageCache.cleanup(config); } diff --git a/lib/workers/pr/changelog/release-notes.ts b/lib/workers/pr/changelog/release-notes.ts index 2ac8bae972b617..6947671a84132a 100644 --- a/lib/workers/pr/changelog/release-notes.ts +++ b/lib/workers/pr/changelog/release-notes.ts @@ -4,7 +4,7 @@ import { linkify } from 'linkify-markdown'; import MarkdownIt from 'markdown-it'; import { logger } from '../../../logger'; -import * as globalCache from '../../../util/cache/global'; +import * as packageCache from '../../../util/cache/package'; import * as runCache from '../../../util/cache/run'; import { GithubHttp } from '../../../util/http/github'; import { GitlabHttp } from '../../../util/http/gitlab'; @@ -354,7 +354,7 @@ export async function addReleaseNotes( for (const v of input.versions) { let releaseNotes: ChangeLogNotes; const cacheKey = getCacheKey(v.version); - releaseNotes = await globalCache.get(cacheNamespace, cacheKey); + releaseNotes = await packageCache.get(cacheNamespace, cacheKey); if (!releaseNotes) { if (input.project.github != null) { releaseNotes = await getReleaseNotesMd( @@ -385,7 +385,7 @@ export async function addReleaseNotes( releaseNotes = { url: v.compare.url }; } const cacheMinutes = 55; - await globalCache.set( + await packageCache.set( cacheNamespace, cacheKey, releaseNotes, diff --git a/lib/workers/pr/changelog/source-github.ts b/lib/workers/pr/changelog/source-github.ts index 33e7a6c0dafc0f..7965b583aee929 100644 --- a/lib/workers/pr/changelog/source-github.ts +++ b/lib/workers/pr/changelog/source-github.ts @@ -2,7 +2,7 @@ import URL from 'url'; import { PLATFORM_TYPE_GITHUB } from '../../../constants/platforms'; import { Release } from '../../../datasource'; import { logger } from '../../../logger'; -import * as globalCache from '../../../util/cache/global'; +import * as packageCache from '../../../util/cache/package'; import * as runCache from '../../../util/cache/run'; import * as hostRules from '../../../util/host-rules'; import { GithubHttp } from '../../../util/http/github'; @@ -150,7 +150,7 @@ export async function getChangeLogJSON({ const prev = validReleases[i - 1]; const next = validReleases[i]; if (include(next.version)) { - let release = await globalCache.get( + let release = await packageCache.get( cacheNamespace, getCacheKey(prev.version, next.version) ); @@ -169,7 +169,7 @@ export async function getChangeLogJSON({ release.compare.url = `${baseUrl}${repository}/compare/${prevHead}...${nextHead}`; } const cacheMinutes = 55; - await globalCache.set( + await packageCache.set( cacheNamespace, getCacheKey(prev.version, next.version), release, diff --git a/lib/workers/pr/changelog/source-gitlab.ts b/lib/workers/pr/changelog/source-gitlab.ts index 4826fe4dd612cb..72540f21542550 100644 --- a/lib/workers/pr/changelog/source-gitlab.ts +++ b/lib/workers/pr/changelog/source-gitlab.ts @@ -1,7 +1,7 @@ import URL from 'url'; import { Release } from '../../../datasource'; import { logger } from '../../../logger'; -import * as globalCache from '../../../util/cache/global'; +import * as packageCache from '../../../util/cache/package'; import * as runCache from '../../../util/cache/run'; import { GitlabHttp } from '../../../util/http/gitlab'; import { regEx } from '../../../util/regex'; @@ -131,7 +131,7 @@ export async function getChangeLogJSON({ const prev = validReleases[i - 1]; const next = validReleases[i]; if (include(next.version)) { - let release = await globalCache.get( + let release = await packageCache.get( cacheNamespace, getCacheKey(prev.version, next.version) ); @@ -149,7 +149,7 @@ export async function getChangeLogJSON({ release.compare.url = `${baseUrl}${repository}/compare/${prevHead}...${nextHead}`; } const cacheMinutes = 55; - await globalCache.set( + await packageCache.set( cacheNamespace, getCacheKey(prev.version, next.version), release, From 9478d6bcfc76f8d2ab5fa81af74e564ad386671d Mon Sep 17 00:00:00 2001 From: Lucas Cimon Date: Thu, 25 Jun 2020 08:34:15 +0200 Subject: [PATCH 16/25] refactor: `extractPy` cache variable in extractSetupFile() (#6543) --- lib/manager/pip_setup/extract.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/manager/pip_setup/extract.ts b/lib/manager/pip_setup/extract.ts index b6985c3fe61632..13baa017e9591a 100644 --- a/lib/manager/pip_setup/extract.ts +++ b/lib/manager/pip_setup/extract.ts @@ -54,12 +54,11 @@ export async function extractSetupFile( ): Promise { const cwd = config.localDir; let cmd: string; - const file = await resolveFile('data/extract.py'); - const args = [`"${file}"`, `"${packageFile}"`]; + extractPy = extractPy || (await resolveFile('data/extract.py')); + const args = [`"${extractPy}"`, `"${packageFile}"`]; if (config.binarySource === BinarySource.Docker) { logger.debug('Running python via docker'); await exec(`docker pull renovate/pip`); - extractPy = extractPy || (await resolveFile('data/extract.py')); cmd = 'docker'; args.unshift( 'run', From b93e072ca9da2c7da34d3c523148279dfae2d73e Mon Sep 17 00:00:00 2001 From: Sergio Zharinov Date: Thu, 25 Jun 2020 10:50:11 +0400 Subject: [PATCH 17/25] refactor: Reorder extractions and lookups (#6578) Co-authored-by: Rhys Arkins --- .../repository/process/extract-update.spec.ts | 5 +- .../repository/process/extract-update.ts | 20 ++++---- lib/workers/repository/process/index.ts | 46 +++++++++++++------ 3 files changed, 47 insertions(+), 24 deletions(-) diff --git a/lib/workers/repository/process/extract-update.spec.ts b/lib/workers/repository/process/extract-update.spec.ts index d599e9ef24cc73..429cd72ea79ec8 100644 --- a/lib/workers/repository/process/extract-update.spec.ts +++ b/lib/workers/repository/process/extract-update.spec.ts @@ -1,6 +1,6 @@ import { mocked } from '../../../../test/util'; import * as _branchify from '../updates/branchify'; -import { extract, update } from './extract-update'; +import { extract, lookup, update } from './extract-update'; jest.mock('./write'); jest.mock('./sort'); @@ -22,7 +22,8 @@ describe('workers/repository/process/extract-update', () => { repoIsOnboarded: true, suppressNotifications: ['deprecationWarningIssues'], }; - const res = await extract(config); + const packageFiles = await extract(config); + const res = await lookup(config, packageFiles); expect(res).toMatchSnapshot(); await expect(update(config, res.branches)).resolves.not.toThrow(); }); diff --git a/lib/workers/repository/process/extract-update.ts b/lib/workers/repository/process/extract-update.ts index 756e10e65ba643..2c4d9bcba3fd21 100644 --- a/lib/workers/repository/process/extract-update.ts +++ b/lib/workers/repository/process/extract-update.ts @@ -1,7 +1,6 @@ import { RenovateConfig } from '../../../config'; import { logger } from '../../../logger'; import { PackageFile } from '../../../manager/common'; -import { addSplit } from '../../../util/split'; import { BranchConfig } from '../../common'; import { extractAllDependencies } from '../extract'; import { branchifyUpgrades } from '../updates/branchify'; @@ -44,18 +43,24 @@ function extractStats(packageFiles: Record): any { return stats; } -export async function extract(config: RenovateConfig): Promise { - logger.debug('extractAndUpdate()'); +export async function extract( + config: RenovateConfig +): Promise> { + logger.debug('extract()'); const packageFiles = await extractAllDependencies(config); const stats = extractStats(packageFiles); logger.info( { baseBranch: config.baseBranch, stats }, `Dependency extraction complete` ); - addSplit( - config.baseBranches?.length ? `extract:${config.baseBranch}` : 'extract' - ); logger.trace({ config: packageFiles }, 'packageFiles'); + return packageFiles; +} + +export async function lookup( + config: RenovateConfig, + packageFiles: Record +): Promise { await fetchUpdates(config, packageFiles); logger.debug({ config: packageFiles }, 'packageFiles with updates'); await raiseDeprecationWarnings(config, packageFiles); @@ -64,9 +69,6 @@ export async function extract(config: RenovateConfig): Promise { packageFiles ); sortBranches(branches); - addSplit( - config.baseBranches?.length ? `lookup:${config.baseBranch}` : 'lookup' - ); return { branches, branchList, packageFiles }; } diff --git a/lib/workers/repository/process/index.ts b/lib/workers/repository/process/index.ts index a9175e69e7920b..8c672386e8dfb2 100644 --- a/lib/workers/repository/process/index.ts +++ b/lib/workers/repository/process/index.ts @@ -1,10 +1,26 @@ import { RenovateConfig, mergeChildConfig } from '../../../config'; import { logger } from '../../../logger'; +import { PackageFile } from '../../../manager/common'; import { platform } from '../../../platform'; +import { addSplit } from '../../../util/split'; import { BranchConfig } from '../../common'; -import { ExtractResult, extract, update } from './extract-update'; +import { ExtractResult, extract, lookup, update } from './extract-update'; import { WriteUpdateResult } from './write'; +async function setBaseBranch( + baseBranch: string, + config: RenovateConfig +): Promise { + logger.debug(`baseBranch: ${baseBranch}`); + const baseBranchConfig = mergeChildConfig(config, { baseBranch }); + if (config.baseBranches.length > 1) { + baseBranchConfig.branchPrefix += `${baseBranch}-`; + baseBranchConfig.hasBaseBranches = true; + } + baseBranchConfig.baseBranchSha = await platform.setBaseBranch(baseBranch); + return baseBranchConfig; +} + export async function extractDependencies( config: RenovateConfig ): Promise { @@ -50,23 +66,27 @@ export async function extractDependencies( }; if (config.baseBranches && config.baseBranches.length) { logger.debug({ baseBranches: config.baseBranches }, 'baseBranches'); + const extracted: Record> = {}; for (const baseBranch of config.baseBranches) { - logger.debug(`baseBranch: ${baseBranch}`); - const baseBranchConfig = mergeChildConfig(config, { baseBranch }); - if (config.baseBranches.length > 1) { - baseBranchConfig.branchPrefix += `${baseBranch}-`; - baseBranchConfig.hasBaseBranches = true; - } - baseBranchConfig.baseBranchSha = await platform.setBaseBranch(baseBranch); - const baseBranchRes = await extract(baseBranchConfig); - res.branches = res.branches.concat(baseBranchRes.branches); - res.branchList = res.branchList.concat(baseBranchRes.branchList); - res.packageFiles = res.packageFiles || baseBranchRes.packageFiles; // Use the first branch + const baseBranchConfig = await setBaseBranch(baseBranch, config); + extracted[baseBranch] = await extract(baseBranchConfig); + } + addSplit('extract'); + for (const baseBranch of config.baseBranches) { + const baseBranchConfig = await setBaseBranch(baseBranch, config); + const packageFiles = extracted[baseBranch]; + const baseBranchRes = await lookup(baseBranchConfig, packageFiles); + res.branches = res.branches.concat(baseBranchRes?.branches); + res.branchList = res.branchList.concat(baseBranchRes?.branchList); + res.packageFiles = res.packageFiles || baseBranchRes?.packageFiles; // Use the first branch } } else { logger.debug('No baseBranches'); - res = await extract(config); + const packageFiles = await extract(config); + addSplit('extract'); + res = await lookup(config, packageFiles); } + addSplit('lookup'); return res; } From 55625a893b3986cf64c72e3f609338013c11df7d Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Thu, 25 Jun 2020 09:23:06 +0200 Subject: [PATCH 18/25] refactor: run cache -> mem cache (#6582) --- lib/datasource/index.ts | 6 +++--- lib/datasource/packagist/index.ts | 6 +++--- lib/manager/bundler/artifacts.ts | 6 +++--- lib/util/cache/memory/index.spec.ts | 18 ++++++++++++++++++ lib/util/cache/{run.ts => memory/index.ts} | 0 lib/util/cache/package/index.ts | 10 +++++----- lib/util/cache/run.spec.ts | 18 ------------------ lib/util/http/index.ts | 10 +++++----- lib/workers/pr/changelog/release-notes.ts | 10 +++++----- lib/workers/pr/changelog/source-github.ts | 6 +++--- lib/workers/pr/changelog/source-gitlab.ts | 6 +++--- lib/workers/repository/init/index.ts | 4 ++-- lib/workers/repository/stats.spec.ts | 8 ++++---- lib/workers/repository/stats.ts | 4 ++-- 14 files changed, 56 insertions(+), 56 deletions(-) create mode 100644 lib/util/cache/memory/index.spec.ts rename lib/util/cache/{run.ts => memory/index.ts} (100%) delete mode 100644 lib/util/cache/run.spec.ts diff --git a/lib/datasource/index.ts b/lib/datasource/index.ts index 1ef183c4e8eec3..92a5d9ac6df134 100644 --- a/lib/datasource/index.ts +++ b/lib/datasource/index.ts @@ -2,7 +2,7 @@ import is from '@sindresorhus/is'; import _ from 'lodash'; import { logger } from '../logger'; import { ExternalHostError } from '../types/errors/external-host-error'; -import * as runCache from '../util/cache/run'; +import * as memCache from '../util/cache/memory'; import { clone } from '../util/clone'; import * as allVersioning from '../versioning'; import datasources from './api.generated'; @@ -190,13 +190,13 @@ function getRawReleases( config.lookupName + config.registryUrls; // By returning a Promise and reusing it, we should only fetch each package at most once - const cachedResult = runCache.get(cacheKey); + const cachedResult = memCache.get(cacheKey); // istanbul ignore if if (cachedResult) { return cachedResult; } const promisedRes = fetchReleases(config); - runCache.set(cacheKey, promisedRes); + memCache.set(cacheKey, promisedRes); return promisedRes; } diff --git a/lib/datasource/packagist/index.ts b/lib/datasource/packagist/index.ts index e75d2602f7a66f..78b8d175dbe5c4 100644 --- a/lib/datasource/packagist/index.ts +++ b/lib/datasource/packagist/index.ts @@ -3,8 +3,8 @@ import URL from 'url'; import pAll from 'p-all'; import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; +import * as memCache from '../../util/cache/memory'; import * as packageCache from '../../util/cache/package'; -import * as runCache from '../../util/cache/run'; import * as hostRules from '../../util/host-rules'; import { Http, HttpOptions } from '../../util/http'; import { GetReleasesConfig, ReleaseResult } from '../common'; @@ -221,13 +221,13 @@ async function getAllPackages(regUrl: string): Promise { function getAllCachedPackages(regUrl: string): Promise { const cacheKey = `packagist-${regUrl}`; - const cachedResult = runCache.get(cacheKey); + const cachedResult = memCache.get(cacheKey); // istanbul ignore if if (cachedResult) { return cachedResult; } const promisedRes = getAllPackages(regUrl); - runCache.set(cacheKey, promisedRes); + memCache.set(cacheKey, promisedRes); return promisedRes; } diff --git a/lib/manager/bundler/artifacts.ts b/lib/manager/bundler/artifacts.ts index def4469669ace9..661c7f119514b0 100644 --- a/lib/manager/bundler/artifacts.ts +++ b/lib/manager/bundler/artifacts.ts @@ -3,7 +3,7 @@ import { BUNDLER_INVALID_CREDENTIALS } from '../../constants/error-messages'; import { logger } from '../../logger'; import { platform } from '../../platform'; import { HostRule } from '../../types'; -import { get, set } from '../../util/cache/run'; +import * as memCache from '../../util/cache/memory'; import { ExecOptions, exec } from '../../util/exec'; import { deleteLocalFile, @@ -74,7 +74,7 @@ export async function updateArtifacts( } = updateArtifact; const { compatibility = {} } = config; logger.debug(`bundler.updateArtifacts(${packageFileName})`); - const existingError = get('bundlerArtifactsError'); + const existingError = memCache.get('bundlerArtifactsError'); // istanbul ignore if if (existingError) { logger.debug('Aborting Bundler artifacts due to previous failed attempt'); @@ -182,7 +182,7 @@ export async function updateArtifacts( 'Gemfile.lock update failed due to missing credentials - skipping branch' ); // Do not generate these PRs because we don't yet support Bundler authentication - set('bundlerArtifactsError', BUNDLER_INVALID_CREDENTIALS); + memCache.set('bundlerArtifactsError', BUNDLER_INVALID_CREDENTIALS); throw new Error(BUNDLER_INVALID_CREDENTIALS); } const resolveMatchRe = new RegExp('\\s+(.*) was resolved to', 'g'); diff --git a/lib/util/cache/memory/index.spec.ts b/lib/util/cache/memory/index.spec.ts new file mode 100644 index 00000000000000..9272efecddabc8 --- /dev/null +++ b/lib/util/cache/memory/index.spec.ts @@ -0,0 +1,18 @@ +import * as memCache from '.'; + +describe('getRepoCache', () => { + it('returns undefined if not init', () => { + expect(memCache.get('key1')).toBeUndefined(); + }); + it('sets and gets repo cache', () => { + memCache.init(); + memCache.set('key2', 'value'); + expect(memCache.get('key2')).toEqual('value'); + }); + it('resets', () => { + memCache.init(); + memCache.set('key3', 'value'); + memCache.reset(); + expect(memCache.get('key3')).toBeUndefined(); + }); +}); diff --git a/lib/util/cache/run.ts b/lib/util/cache/memory/index.ts similarity index 100% rename from lib/util/cache/run.ts rename to lib/util/cache/memory/index.ts diff --git a/lib/util/cache/package/index.ts b/lib/util/cache/package/index.ts index d77fdcd8be412e..6962b73c6ad2b1 100644 --- a/lib/util/cache/package/index.ts +++ b/lib/util/cache/package/index.ts @@ -1,5 +1,5 @@ import { RenovateConfig } from '../../../config/common'; -import * as runCache from '../run'; +import * as memCache from '../memory'; import { PackageCache } from './common'; import * as fileCache from './file'; import * as redisCache from './redis'; @@ -15,10 +15,10 @@ export function get(namespace: string, key: string): Promise { return undefined; } const globalKey = getGlobalKey(namespace, key); - if (!runCache.get(globalKey)) { - runCache.set(globalKey, cacheProxy.get(namespace, key)); + if (!memCache.get(globalKey)) { + memCache.set(globalKey, cacheProxy.get(namespace, key)); } - return runCache.get(globalKey); + return memCache.get(globalKey); } export function set( @@ -31,7 +31,7 @@ export function set( return undefined; } const globalKey = getGlobalKey(namespace, key); - runCache.set(globalKey, value); + memCache.set(globalKey, value); return cacheProxy.set(namespace, key, value, minutes); } diff --git a/lib/util/cache/run.spec.ts b/lib/util/cache/run.spec.ts deleted file mode 100644 index df1cba40ebb693..00000000000000 --- a/lib/util/cache/run.spec.ts +++ /dev/null @@ -1,18 +0,0 @@ -import * as runCache from './run'; - -describe('getRepoCache', () => { - it('returns undefined if not init', () => { - expect(runCache.get('key1')).toBeUndefined(); - }); - it('sets and gets repo cache', () => { - runCache.init(); - runCache.set('key2', 'value'); - expect(runCache.get('key2')).toEqual('value'); - }); - it('resets', () => { - runCache.init(); - runCache.set('key3', 'value'); - runCache.reset(); - expect(runCache.get('key3')).toBeUndefined(); - }); -}); diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts index 43da6e0cb8b371..c140432f724c77 100644 --- a/lib/util/http/index.ts +++ b/lib/util/http/index.ts @@ -2,7 +2,7 @@ import crypto from 'crypto'; import URL from 'url'; import got from 'got'; import { ExternalHostError } from '../../types/errors/external-host-error'; -import * as runCache from '../cache/run'; +import * as memCache from '../cache/memory'; import { clone } from '../clone'; import { applyAuthorization } from './auth'; import { applyHostRules } from './host-rules'; @@ -115,7 +115,7 @@ export class Http { .digest('hex'); if (options.method === 'get' && options.useCache !== false) { // return from cache if present - const cachedRes = runCache.get(cacheKey); + const cachedRes = memCache.get(cacheKey); // istanbul ignore if if (cachedRes) { return resolveResponse(cachedRes, options); @@ -124,16 +124,16 @@ export class Http { const startTime = Date.now(); const promisedRes = got(url, options); if (options.method === 'get') { - runCache.set(cacheKey, promisedRes); // always set if it's a get + memCache.set(cacheKey, promisedRes); // always set if it's a get } const res = await resolveResponse(promisedRes, options); - const httpRequests = runCache.get('http-requests') || []; + const httpRequests = memCache.get('http-requests') || []; httpRequests.push({ method: options.method, url, duration: Date.now() - startTime, }); - runCache.set('http-requests', httpRequests); + memCache.set('http-requests', httpRequests); return res; } diff --git a/lib/workers/pr/changelog/release-notes.ts b/lib/workers/pr/changelog/release-notes.ts index 6947671a84132a..6fcb94c6cb3a98 100644 --- a/lib/workers/pr/changelog/release-notes.ts +++ b/lib/workers/pr/changelog/release-notes.ts @@ -4,8 +4,8 @@ import { linkify } from 'linkify-markdown'; import MarkdownIt from 'markdown-it'; import { logger } from '../../../logger'; +import * as memCache from '../../../util/cache/memory'; import * as packageCache from '../../../util/cache/package'; -import * as runCache from '../../../util/cache/run'; import { GithubHttp } from '../../../util/http/github'; import { GitlabHttp } from '../../../util/http/gitlab'; import { ChangeLogNotes, ChangeLogResult } from './common'; @@ -83,13 +83,13 @@ export function getCachedReleaseList( repository: string ): Promise { const cacheKey = `getReleaseList-${apiBaseUrl}-${repository}`; - const cachedResult = runCache.get(cacheKey); + const cachedResult = memCache.get(cacheKey); // istanbul ignore if if (cachedResult) { return cachedResult; } const promisedRes = getReleaseList(apiBaseUrl, repository); - runCache.set(cacheKey, promisedRes); + memCache.set(cacheKey, promisedRes); return promisedRes; } @@ -254,13 +254,13 @@ export async function getReleaseNotesMdFile( apiBaseUrl: string ): Promise<{ changelogFile: string; changelogMd: string }> | null { const cacheKey = `getReleaseNotesMdFile-${repository}-${apiBaseUrl}`; - const cachedResult = runCache.get(cacheKey); + const cachedResult = memCache.get(cacheKey); // istanbul ignore if if (cachedResult !== undefined) { return cachedResult; } const promisedRes = getReleaseNotesMdFileInner(repository, apiBaseUrl); - runCache.set(cacheKey, promisedRes); + memCache.set(cacheKey, promisedRes); return promisedRes; } diff --git a/lib/workers/pr/changelog/source-github.ts b/lib/workers/pr/changelog/source-github.ts index 7965b583aee929..2a36a600ecf690 100644 --- a/lib/workers/pr/changelog/source-github.ts +++ b/lib/workers/pr/changelog/source-github.ts @@ -2,8 +2,8 @@ import URL from 'url'; import { PLATFORM_TYPE_GITHUB } from '../../../constants/platforms'; import { Release } from '../../../datasource'; import { logger } from '../../../logger'; +import * as memCache from '../../../util/cache/memory'; import * as packageCache from '../../../util/cache/package'; -import * as runCache from '../../../util/cache/run'; import * as hostRules from '../../../util/host-rules'; import { GithubHttp } from '../../../util/http/github'; import * as allVersioning from '../../../versioning'; @@ -47,13 +47,13 @@ async function getTags( repository: string ): Promise { const cacheKey = `getTags-${endpoint}-${repository}`; - const cachedResult = runCache.get(cacheKey); + const cachedResult = memCache.get(cacheKey); // istanbul ignore if if (cachedResult !== undefined) { return cachedResult; } const promisedRes = getTagsInner(endpoint, repository); - runCache.set(cacheKey, promisedRes); + memCache.set(cacheKey, promisedRes); return promisedRes; } diff --git a/lib/workers/pr/changelog/source-gitlab.ts b/lib/workers/pr/changelog/source-gitlab.ts index 72540f21542550..e2f23a431c718f 100644 --- a/lib/workers/pr/changelog/source-gitlab.ts +++ b/lib/workers/pr/changelog/source-gitlab.ts @@ -1,8 +1,8 @@ import URL from 'url'; import { Release } from '../../../datasource'; import { logger } from '../../../logger'; +import * as memCache from '../../../util/cache/memory'; import * as packageCache from '../../../util/cache/package'; -import * as runCache from '../../../util/cache/run'; import { GitlabHttp } from '../../../util/http/gitlab'; import { regEx } from '../../../util/regex'; import * as allVersioning from '../../../versioning'; @@ -52,13 +52,13 @@ async function getTags( repository: string ): Promise { const cacheKey = `getTags-${endpoint}-${versionScheme}-${repository}`; - const cachedResult = runCache.get(cacheKey); + const cachedResult = memCache.get(cacheKey); // istanbul ignore if if (cachedResult !== undefined) { return cachedResult; } const promisedRes = getTagsInner(endpoint, versionScheme, repository); - runCache.set(cacheKey, promisedRes); + memCache.set(cacheKey, promisedRes); return promisedRes; } diff --git a/lib/workers/repository/init/index.ts b/lib/workers/repository/init/index.ts index 5b3688c960e300..f0c901d5055789 100644 --- a/lib/workers/repository/init/index.ts +++ b/lib/workers/repository/init/index.ts @@ -1,7 +1,7 @@ import { RenovateConfig } from '../../../config'; import { logger } from '../../../logger'; import { platform } from '../../../platform'; -import * as runCache from '../../../util/cache/run'; +import * as memCache from '../../../util/cache/memory'; import { checkIfConfigured } from '../configured'; import { checkOnboardingBranch } from '../onboarding/branch'; import { initApis } from './apis'; @@ -11,7 +11,7 @@ import { detectSemanticCommits } from './semantic'; import { detectVulnerabilityAlerts } from './vulnerability'; export async function initRepo(input: RenovateConfig): Promise { - runCache.init(); + memCache.init(); let config: RenovateConfig = { ...input, errors: [], diff --git a/lib/workers/repository/stats.spec.ts b/lib/workers/repository/stats.spec.ts index a391efba56c2ae..a62796aa981944 100644 --- a/lib/workers/repository/stats.spec.ts +++ b/lib/workers/repository/stats.spec.ts @@ -1,14 +1,14 @@ -import * as runCache_ from '../../util/cache/run'; +import * as memCache_ from '../../util/cache/memory'; import { printRequestStats } from './stats'; -jest.mock('../../util/cache/run'); +jest.mock('../../util/cache/memory'); -const runCache: any = runCache_ as any; +const memCache: any = memCache_ as any; describe('workers/repository/stats', () => { describe('printRequestStats()', () => { it('runs', () => { - runCache.get = jest.fn(() => [ + memCache.get = jest.fn(() => [ { method: 'get', url: 'https://api.github.com/api/v3/user', diff --git a/lib/workers/repository/stats.ts b/lib/workers/repository/stats.ts index 167a8b659df7b3..2cf727b4314042 100644 --- a/lib/workers/repository/stats.ts +++ b/lib/workers/repository/stats.ts @@ -1,9 +1,9 @@ import URL from 'url'; import { logger } from '../../logger'; -import * as runCache from '../../util/cache/run'; +import * as memCache from '../../util/cache/memory'; export function printRequestStats(): void { - const httpRequests = runCache.get('http-requests'); + const httpRequests = memCache.get('http-requests'); if (!httpRequests) { return; } From b69d4f29e5ad4153be15a928ed3820a17d2cd07b Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Thu, 25 Jun 2020 09:34:41 +0200 Subject: [PATCH 19/25] refactor: add try/catch and logging to datasource wrapper --- lib/datasource/index.spec.ts | 16 ++++++------ lib/datasource/index.ts | 47 +++++++++++++++++++++++++++--------- 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/lib/datasource/index.spec.ts b/lib/datasource/index.spec.ts index 160a5bb30f50a0..cce29d3bfb255b 100644 --- a/lib/datasource/index.spec.ts +++ b/lib/datasource/index.spec.ts @@ -143,20 +143,20 @@ describe('datasource/index', () => { }) ).rejects.toThrow(EXTERNAL_HOST_ERROR); }); - it('hunts registries and passes on error', async () => { + it('hunts registries and returns null', async () => { packagistDatasource.getReleases.mockImplementationOnce(() => { throw new Error('a'); }); packagistDatasource.getReleases.mockImplementationOnce(() => { throw new Error('b'); }); - await expect( - datasource.getPkgReleases({ + expect( + await datasource.getPkgReleases({ datasource: datasourcePackagist.id, depName: 'something', registryUrls: ['https://reg1.com', 'https://reg2.io'], }) - ).rejects.toThrow('b'); + ).toBeNull(); }); it('merges registries and returns success', async () => { mavenDatasource.getReleases.mockResolvedValueOnce({ @@ -185,20 +185,20 @@ describe('datasource/index', () => { }) ).rejects.toThrow(EXTERNAL_HOST_ERROR); }); - it('merges registries and passes on error', async () => { + it('merges registries and returns null for error', async () => { mavenDatasource.getReleases.mockImplementationOnce(() => { throw new Error('a'); }); mavenDatasource.getReleases.mockImplementationOnce(() => { throw new Error('b'); }); - await expect( - datasource.getPkgReleases({ + expect( + await datasource.getPkgReleases({ datasource: datasourceMaven.id, depName: 'something', registryUrls: ['https://reg1.com', 'https://reg2.io'], }) - ).rejects.toThrow('b'); + ).toBeNull(); }); it('trims sourceUrl', async () => { npmDatasource.getReleases.mockResolvedValue({ diff --git a/lib/datasource/index.ts b/lib/datasource/index.ts index 92a5d9ac6df134..bc3fe6d73bcb2a 100644 --- a/lib/datasource/index.ts +++ b/lib/datasource/index.ts @@ -30,6 +30,18 @@ function load(datasource: string): Promise { type GetReleasesInternalConfig = GetReleasesConfig & GetPkgReleasesConfig; +// istanbul ignore next +function logError(datasource, lookupName, err): void { + const { statusCode, url } = err; + if (statusCode === 404) { + logger.debug({ datasource, lookupName, url }, 'Datasource 404'); + } else if (statusCode === 401 || statusCode === 403) { + logger.debug({ datasource, lookupName, url }, 'Datasource unauthorized'); + } else { + logger.debug({ datasource, lookupName, err }, 'Datasource unknown error'); + } +} + async function getRegistryReleases( datasource, config: GetReleasesConfig, @@ -39,7 +51,7 @@ async function getRegistryReleases( return res; } -function firstRegistry( +async function firstRegistry( config: GetReleasesInternalConfig, datasource: Datasource, registryUrls: string[] @@ -51,7 +63,16 @@ function firstRegistry( ); } const registryUrl = registryUrls[0]; - return getRegistryReleases(datasource, config, registryUrl); + try { + const res = await getRegistryReleases(datasource, config, registryUrl); + return res; + } catch (err) /* istanbul ignore next */ { + if (err instanceof ExternalHostError) { + throw err; + } + logError(datasource.id, config.lookupName, err); + return null; + } } async function huntRegistries( @@ -76,11 +97,13 @@ async function huntRegistries( logger.trace({ err }, 'datasource hunt failure'); } } - if (res === undefined && datasourceError) { - // if we failed to get a result and also got an error then throw it - throw datasourceError; + if (res) { + return res; + } + if (datasourceError) { + logError(datasource.id, config.lookupName, datasourceError); } - return res; + return null; } async function mergeRegistries( @@ -108,10 +131,6 @@ async function mergeRegistries( logger.trace({ err }, 'datasource merge failure'); } } - if (combinedRes === undefined && datasourceError) { - // if we failed to get a result and also got an error then throw it - throw datasourceError; - } // De-duplicate releases if (combinedRes?.releases?.length) { const seenVersions = new Set(); @@ -123,7 +142,13 @@ async function mergeRegistries( return true; }); } - return combinedRes; + if (combinedRes) { + return combinedRes; + } + if (datasourceError) { + logError(datasource.id, config.lookupName, datasourceError); + } + return null; } function resolveRegistryUrls( From 7902ba06be4bd4b4c1b0530f1852b920183e5f06 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Thu, 25 Jun 2020 10:29:45 +0200 Subject: [PATCH 20/25] refactor: move datasource error catching --- lib/datasource/index.ts | 74 ++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 38 deletions(-) diff --git a/lib/datasource/index.ts b/lib/datasource/index.ts index bc3fe6d73bcb2a..b8b2d2817ca14f 100644 --- a/lib/datasource/index.ts +++ b/lib/datasource/index.ts @@ -63,16 +63,7 @@ async function firstRegistry( ); } const registryUrl = registryUrls[0]; - try { - const res = await getRegistryReleases(datasource, config, registryUrl); - return res; - } catch (err) /* istanbul ignore next */ { - if (err instanceof ExternalHostError) { - throw err; - } - logError(datasource.id, config.lookupName, err); - return null; - } + return getRegistryReleases(datasource, config, registryUrl); } async function huntRegistries( @@ -81,7 +72,7 @@ async function huntRegistries( registryUrls: string[] ): Promise { let res: ReleaseResult; - let datasourceError; + let caughtError; for (const registryUrl of registryUrls) { try { res = await getRegistryReleases(datasource, config, registryUrl); @@ -93,15 +84,15 @@ async function huntRegistries( throw err; } // We'll always save the last-thrown error - datasourceError = err; + caughtError = err; logger.trace({ err }, 'datasource hunt failure'); } } if (res) { return res; } - if (datasourceError) { - logError(datasource.id, config.lookupName, datasourceError); + if (caughtError) { + throw caughtError; } return null; } @@ -112,7 +103,7 @@ async function mergeRegistries( registryUrls: string[] ): Promise { let combinedRes: ReleaseResult; - let datasourceError; + let caughtError; for (const registryUrl of registryUrls) { try { const res = await getRegistryReleases(datasource, config, registryUrl); @@ -127,7 +118,7 @@ async function mergeRegistries( throw err; } // We'll always save the last-thrown error - datasourceError = err; + caughtError = err; logger.trace({ err }, 'datasource merge failure'); } } @@ -145,8 +136,8 @@ async function mergeRegistries( if (combinedRes) { return combinedRes; } - if (datasourceError) { - logError(datasource.id, config.lookupName, datasourceError); + if (caughtError) { + throw caughtError; } return null; } @@ -176,28 +167,35 @@ async function fetchReleases( } const datasource = await load(datasourceName); const registryUrls = resolveRegistryUrls(datasource, config.registryUrls); - let dep: ReleaseResult; - if (datasource.registryStrategy) { - // istanbul ignore if - if (!registryUrls.length) { - logger.warn( - { datasource: datasourceName, depName: config.depName }, - 'Missing registryUrls for registryStrategy' - ); - return null; + let dep: ReleaseResult = null; + try { + if (datasource.registryStrategy) { + // istanbul ignore if + if (!registryUrls.length) { + logger.warn( + { datasource: datasourceName, depName: config.depName }, + 'Missing registryUrls for registryStrategy' + ); + return null; + } + if (datasource.registryStrategy === 'first') { + dep = await firstRegistry(config, datasource, registryUrls); + } else if (datasource.registryStrategy === 'hunt') { + dep = await huntRegistries(config, datasource, registryUrls); + } else if (datasource.registryStrategy === 'merge') { + dep = await mergeRegistries(config, datasource, registryUrls); + } + } else { + dep = await datasource.getReleases({ + ...config, + registryUrls, + }); } - if (datasource.registryStrategy === 'first') { - dep = await firstRegistry(config, datasource, registryUrls); - } else if (datasource.registryStrategy === 'hunt') { - dep = await huntRegistries(config, datasource, registryUrls); - } else if (datasource.registryStrategy === 'merge') { - dep = await mergeRegistries(config, datasource, registryUrls); + } catch (err) { + if (err instanceof ExternalHostError) { + throw err; } - } else { - dep = await datasource.getReleases({ - ...config, - registryUrls, - }); + logError(datasource.id, config.lookupName, err); } if (!dep || _.isEqual(dep, { releases: [] })) { return null; From 14f92702597142d75c70932e2e01989ca7f86e9c Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Thu, 25 Jun 2020 14:07:03 +0200 Subject: [PATCH 21/25] refactor: remove unnecessary catch code in datasources (#6583) --- lib/datasource/cdnjs/index.ts | 9 +- lib/datasource/crate/index.ts | 9 +- lib/datasource/dart/index.ts | 12 +- lib/datasource/docker/index.ts | 26 +-- lib/datasource/galaxy/index.ts | 2 +- lib/datasource/git-refs/index.ts | 147 ++++++++-------- lib/datasource/git-submodules/index.ts | 42 ++--- lib/datasource/github-releases/index.ts | 20 +-- lib/datasource/github-tags/index.ts | 28 ++- lib/datasource/gitlab-tags/index.ts | 32 ++-- lib/datasource/go/index.ts | 49 ++---- lib/datasource/gradle-version/index.ts | 4 +- lib/datasource/helm/index.ts | 32 +--- lib/datasource/hex/index.ts | 13 +- lib/datasource/index.ts | 8 +- lib/datasource/maven/index.ts | 20 +-- lib/datasource/nuget/v2.ts | 74 ++++---- lib/datasource/nuget/v3.ts | 102 ++++++----- lib/datasource/orb/index.ts | 62 +++---- lib/datasource/packagist/index.ts | 104 ++++-------- lib/datasource/pypi/index.ts | 187 +++++++++------------ lib/datasource/rubygems/errors.ts | 17 -- lib/datasource/rubygems/get.ts | 96 ++++------- lib/datasource/terraform-module/index.ts | 16 +- lib/datasource/terraform-provider/index.ts | 81 ++++----- 25 files changed, 431 insertions(+), 761 deletions(-) delete mode 100644 lib/datasource/rubygems/errors.ts diff --git a/lib/datasource/cdnjs/index.ts b/lib/datasource/cdnjs/index.ts index a994a6b8c639e8..58f912c6454202 100644 --- a/lib/datasource/cdnjs/index.ts +++ b/lib/datasource/cdnjs/index.ts @@ -1,4 +1,3 @@ -import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; import { Http } from '../../util/http'; import { CachePromise, cacheAble } from '../cache'; @@ -57,11 +56,9 @@ export async function getReleases({ } return result; } catch (err) { - if (err.statusCode === 404) { - logger.debug({ library }, 'cdnjs library not found'); - return null; + if (err.statusCode !== 404) { + throw new ExternalHostError(err); } - // Throw an ExternalHostError for all other types of errors - throw new ExternalHostError(err); + throw err; } } diff --git a/lib/datasource/crate/index.ts b/lib/datasource/crate/index.ts index 9715b4eab18353..324dad6e0065f8 100644 --- a/lib/datasource/crate/index.ts +++ b/lib/datasource/crate/index.ts @@ -1,4 +1,3 @@ -import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; import * as packageCache from '../../util/cache/package'; import { Http } from '../../util/http'; @@ -66,18 +65,12 @@ export async function getReleases({ await packageCache.set(cacheNamespace, cacheKey, result, cacheMinutes); return result; } catch (err) { - if (err.statusCode === 404 || err.code === 'ENOTFOUND') { - logger.debug({ lookupName }, `Dependency lookup failure: not found`); - logger.debug({ err }, 'Crate lookup error'); - return null; - } if ( err.statusCode === 429 || (err.statusCode >= 500 && err.statusCode < 600) ) { throw new ExternalHostError(err); } - logger.warn({ err, lookupName }, 'crates.io lookup failure: Unknown error'); - return null; + throw err; } } diff --git a/lib/datasource/dart/index.ts b/lib/datasource/dart/index.ts index 241394373b5d7b..94f6dc4d218ae0 100644 --- a/lib/datasource/dart/index.ts +++ b/lib/datasource/dart/index.ts @@ -1,4 +1,3 @@ -import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; import { Http, HttpResponse } from '../../util/http'; import { GetReleasesConfig, ReleaseResult } from '../common'; @@ -25,22 +24,13 @@ export async function getReleases({ try { raw = await http.getJson(pkgUrl); } catch (err) { - if (err.statusCode === 404 || err.code === 'ENOTFOUND') { - logger.debug({ lookupName }, `Dependency lookup failure: not found`); - logger.debug({ err }, 'Dart lookup error'); - return null; - } if ( err.statusCode === 429 || (err.statusCode >= 500 && err.statusCode < 600) ) { throw new ExternalHostError(err); } - logger.warn( - { err, lookupName }, - 'pub.dartlang.org lookup failure: Unknown error' - ); - return null; + throw err; } const body = raw && raw.body; diff --git a/lib/datasource/docker/index.ts b/lib/datasource/docker/index.ts index 4af0da52eac34e..bb7d5744834710 100644 --- a/lib/datasource/docker/index.ts +++ b/lib/datasource/docker/index.ts @@ -417,25 +417,12 @@ async function getTags( if (err instanceof ExternalHostError) { throw err; } - logger.debug( - { - err, - }, - 'docker.getTags() error' - ); if (err.statusCode === 404 && !repository.includes('/')) { logger.debug( `Retrying Tags for ${registry}/${repository} using library/ prefix` ); return getTags(registry, 'library/' + repository); } - if (err.statusCode === 401 || err.statusCode === 403) { - logger.debug( - { registry, dockerRepository: repository, err }, - 'Not authorised to look up docker tags' - ); - return null; - } // prettier-ignore if (err.statusCode === 429 && registry.endsWith('docker.io')) { // lgtm [js/incomplete-url-substring-sanitization] logger.warn( @@ -451,18 +438,7 @@ async function getTags( ); throw new ExternalHostError(err); } - if (err.code === 'ETIMEDOUT') { - logger.debug( - { registry }, - 'Timeout when attempting to connect to docker registry' - ); - return null; - } - logger.warn( - { registry, dockerRepository: repository, err }, - 'Error getting docker image tags' - ); - return null; + throw err; } } diff --git a/lib/datasource/galaxy/index.ts b/lib/datasource/galaxy/index.ts index ed01e05fe8a442..48ef50b7c782df 100644 --- a/lib/datasource/galaxy/index.ts +++ b/lib/datasource/galaxy/index.ts @@ -99,6 +99,6 @@ export async function getReleases({ ) { throw new ExternalHostError(err); } - return null; + throw err; } } diff --git a/lib/datasource/git-refs/index.ts b/lib/datasource/git-refs/index.ts index 50820ee5920950..c872bc33f23336 100644 --- a/lib/datasource/git-refs/index.ts +++ b/lib/datasource/git-refs/index.ts @@ -1,5 +1,4 @@ import simpleGit from 'simple-git/promise'; -import { logger } from '../../logger'; import * as packageCache from '../../util/cache/package'; import * as semver from '../../versioning/semver'; import { DigestConfig, GetReleasesConfig, ReleaseResult } from '../common'; @@ -21,90 +20,80 @@ export async function getRawRefs({ lookupName, }: GetReleasesConfig): Promise { const git = simpleGit(); - try { - const cacheNamespace = 'git-raw-refs'; - - const cachedResult = await packageCache.get( - cacheNamespace, - lookupName - ); - /* istanbul ignore next line */ - if (cachedResult) { - return cachedResult; - } - - // fetch remote tags - const lsRemote = await git.listRemote([lookupName]); - if (!lsRemote) { - return null; - } - - const refMatch = /(?.*?)\s+refs\/(?.*?)\/(?.*)/; - const headMatch = /(?.*?)\s+HEAD/; - - const refs = lsRemote - .trim() - .split('\n') - .map((line) => line.trim()) - .map((line) => { - let match = refMatch.exec(line); - if (match) { - return { - type: match.groups.type, - value: match.groups.value, - hash: match.groups.hash, - }; - } - match = headMatch.exec(line); - if (match) { - return { - type: '', - value: 'HEAD', - hash: match.groups.hash, - }; - } - // istanbul ignore next - return null; - }) - .filter(Boolean) - .filter((ref) => ref.type !== 'pull' && !ref.value.endsWith('^{}')); - await packageCache.set(cacheNamespace, lookupName, refs, cacheMinutes); - return refs; - } catch (err) { - logger.info({ err }, `Git-Raw-Refs lookup error in ${lookupName}`); + const cacheNamespace = 'git-raw-refs'; + + const cachedResult = await packageCache.get( + cacheNamespace, + lookupName + ); + /* istanbul ignore next line */ + if (cachedResult) { + return cachedResult; } - return null; + + // fetch remote tags + const lsRemote = await git.listRemote([lookupName]); + if (!lsRemote) { + return null; + } + + const refMatch = /(?.*?)\s+refs\/(?.*?)\/(?.*)/; + const headMatch = /(?.*?)\s+HEAD/; + + const refs = lsRemote + .trim() + .split('\n') + .map((line) => line.trim()) + .map((line) => { + let match = refMatch.exec(line); + if (match) { + return { + type: match.groups.type, + value: match.groups.value, + hash: match.groups.hash, + }; + } + match = headMatch.exec(line); + if (match) { + return { + type: '', + value: 'HEAD', + hash: match.groups.hash, + }; + } + // istanbul ignore next + return null; + }) + .filter(Boolean) + .filter((ref) => ref.type !== 'pull' && !ref.value.endsWith('^{}')); + await packageCache.set(cacheNamespace, lookupName, refs, cacheMinutes); + return refs; } export async function getReleases({ lookupName, }: GetReleasesConfig): Promise { - try { - const rawRefs: RawRefs[] = await getRawRefs({ lookupName }); - - const refs = rawRefs - .filter((ref) => ref.type === 'tags' || ref.type === 'heads') - .map((ref) => ref.value) - .filter((ref) => semver.isVersion(ref)); - - const uniqueRefs = [...new Set(refs)]; - - const sourceUrl = lookupName.replace(/\.git$/, '').replace(/\/$/, ''); - - const result: ReleaseResult = { - sourceUrl, - releases: uniqueRefs.map((ref) => ({ - version: ref, - gitRef: ref, - newDigest: rawRefs.find((rawRef) => rawRef.value === ref).hash, - })), - }; - - return result; - } catch (err) { - logger.error({ err }, `Git-Refs lookup error in ${lookupName}`); - } - return null; + const rawRefs: RawRefs[] = await getRawRefs({ lookupName }); + + const refs = rawRefs + .filter((ref) => ref.type === 'tags' || ref.type === 'heads') + .map((ref) => ref.value) + .filter((ref) => semver.isVersion(ref)); + + const uniqueRefs = [...new Set(refs)]; + + const sourceUrl = lookupName.replace(/\.git$/, '').replace(/\/$/, ''); + + const result: ReleaseResult = { + sourceUrl, + releases: uniqueRefs.map((ref) => ({ + version: ref, + gitRef: ref, + newDigest: rawRefs.find((rawRef) => rawRef.value === ref).hash, + })), + }; + + return result; } export async function getDigest( diff --git a/lib/datasource/git-submodules/index.ts b/lib/datasource/git-submodules/index.ts index 4a812a3fb748f9..22a8b6adeab013 100644 --- a/lib/datasource/git-submodules/index.ts +++ b/lib/datasource/git-submodules/index.ts @@ -1,7 +1,6 @@ import { URL } from 'url'; import Git from 'simple-git/promise'; -import { logger } from '../../logger'; import * as packageCache from '../../util/cache/package'; import { DigestConfig, GetReleasesConfig, ReleaseResult } from '../common'; @@ -23,31 +22,26 @@ export async function getReleases({ } const git = Git(); - try { - const newHash = ( - await git.listRemote(['--refs', registryUrls[0], registryUrls[1]]) - ) - .trim() - .split(/\t/)[0]; + const newHash = ( + await git.listRemote(['--refs', registryUrls[0], registryUrls[1]]) + ) + .trim() + .split(/\t/)[0]; - const sourceUrl = new URL(registryUrls[0]); - sourceUrl.username = ''; + const sourceUrl = new URL(registryUrls[0]); + sourceUrl.username = ''; - const result = { - sourceUrl: sourceUrl.href, - releases: [ - { - version: newHash, - }, - ], - }; - const cacheMinutes = 60; - await packageCache.set(cacheNamespace, cacheKey, result, cacheMinutes); - return result; - } catch (err) { - logger.debug({ err }, `Git-SubModules lookup error in ${lookupName}`); - } - return null; + const result = { + sourceUrl: sourceUrl.href, + releases: [ + { + version: newHash, + }, + ], + }; + const cacheMinutes = 60; + await packageCache.set(cacheNamespace, cacheKey, result, cacheMinutes); + return result; } export const getDigest = ( diff --git a/lib/datasource/github-releases/index.ts b/lib/datasource/github-releases/index.ts index 438f6509ac5f6a..de2ecd435b16a3 100644 --- a/lib/datasource/github-releases/index.ts +++ b/lib/datasource/github-releases/index.ts @@ -1,4 +1,3 @@ -import { logger } from '../../logger'; import * as packageCache from '../../util/cache/package'; import { GithubHttp } from '../../util/http/github'; import { GetReleasesConfig, ReleaseResult } from '../common'; @@ -27,7 +26,6 @@ type GithubRelease = { export async function getReleases({ lookupName: repo, }: GetReleasesConfig): Promise { - let githubReleases: GithubRelease[]; const cachedResult = await packageCache.get( cacheNamespace, repo @@ -36,19 +34,11 @@ export async function getReleases({ if (cachedResult) { return cachedResult; } - try { - const url = `https://api.github.com/repos/${repo}/releases?per_page=100`; - const res = await http.getJson(url, { - paginate: true, - }); - githubReleases = res.body; - } catch (err) /* istanbul ignore next */ { - logger.debug({ repo, err }, 'Error retrieving from github'); - } - // istanbul ignore if - if (!githubReleases) { - return null; - } + const url = `https://api.github.com/repos/${repo}/releases?per_page=100`; + const res = await http.getJson(url, { + paginate: true, + }); + const githubReleases = res.body; const dependency: ReleaseResult = { sourceUrl: 'https://github.com/' + repo, releases: null, diff --git a/lib/datasource/github-tags/index.ts b/lib/datasource/github-tags/index.ts index d2a385a42bafac..2e0877a9f70b10 100644 --- a/lib/datasource/github-tags/index.ts +++ b/lib/datasource/github-tags/index.ts @@ -121,7 +121,6 @@ export async function getDigest( export async function getReleases({ lookupName: repo, }: GetReleasesConfig): Promise { - let versions: string[]; const cachedResult = await packageCache.get( cacheNamespace, getCacheKey(repo, 'tags') @@ -130,24 +129,17 @@ export async function getReleases({ if (cachedResult) { return cachedResult; } - try { - // tag - const url = `https://api.github.com/repos/${repo}/tags?per_page=100`; - type GitHubTag = { - name: string; - }[]; + // tag + const url = `https://api.github.com/repos/${repo}/tags?per_page=100`; + type GitHubTag = { + name: string; + }[]; - versions = ( - await http.getJson(url, { - paginate: true, - }) - ).body.map((o) => o.name); - } catch (err) { - logger.debug({ repo, err }, 'Error retrieving from github'); - } - if (!versions) { - return null; - } + const versions = ( + await http.getJson(url, { + paginate: true, + }) + ).body.map((o) => o.name); const dependency: ReleaseResult = { sourceUrl: 'https://github.com/' + repo, releases: null, diff --git a/lib/datasource/gitlab-tags/index.ts b/lib/datasource/gitlab-tags/index.ts index 59bff4283b83e6..22d481253e7b34 100644 --- a/lib/datasource/gitlab-tags/index.ts +++ b/lib/datasource/gitlab-tags/index.ts @@ -1,5 +1,4 @@ import URL from 'url'; -import { logger } from '../../logger'; import * as packageCache from '../../util/cache/package'; import { GitlabHttp } from '../../util/http/gitlab'; import { GetReleasesConfig, ReleaseResult } from '../common'; @@ -27,7 +26,6 @@ export async function getReleases({ registryUrl: depHost, lookupName: repo, }: GetReleasesConfig): Promise { - let gitlabTags: GitlabTag[]; const cachedResult = await packageCache.get( cacheNamespace, getCacheKey(depHost, repo) @@ -39,27 +37,17 @@ export async function getReleases({ const urlEncodedRepo = encodeURIComponent(repo); - try { - // tag - const url = URL.resolve( - depHost, - `/api/v4/projects/${urlEncodedRepo}/repository/tags?per_page=100` - ); - - gitlabTags = ( - await gitlabApi.getJson(url, { - paginate: true, - }) - ).body; - } catch (err) { - // istanbul ignore next - logger.debug({ repo, err }, 'Error retrieving from Gitlab'); - } + // tag + const url = URL.resolve( + depHost, + `/api/v4/projects/${urlEncodedRepo}/repository/tags?per_page=100` + ); - // istanbul ignore if - if (!gitlabTags) { - return null; - } + const gitlabTags = ( + await gitlabApi.getJson(url, { + paginate: true, + }) + ).body; const dependency: ReleaseResult = { sourceUrl: URL.resolve(depHost, repo), diff --git a/lib/datasource/go/index.ts b/lib/datasource/go/index.ts index 0e658bacf26751..c4411f08dcfeb6 100644 --- a/lib/datasource/go/index.ts +++ b/lib/datasource/go/index.ts @@ -33,40 +33,25 @@ async function getDatasource(goModule: string): Promise { }; } const pkgUrl = `https://${goModule}?go-get=1`; - try { - const res = (await http.get(pkgUrl)).body; - const sourceMatch = regEx( - `(url, { - body, - }) - ).body.data.orb; - if (!res) { - logger.debug({ lookupName }, 'Failed to look up orb'); - return null; - } - // Simplify response before caching and returning - const dep: ReleaseResult = { - name: lookupName, - versions: {}, - releases: null, - }; - if (res.homeUrl && res.homeUrl.length) { - dep.homepage = res.homeUrl; - } - dep.homepage = - dep.homepage || `https://circleci.com/orbs/registry/orb/${lookupName}`; - dep.releases = res.versions.map(({ version, createdAt }) => ({ - version, - releaseTimestamp: createdAt || null, - })); - logger.trace({ dep }, 'dep'); - const cacheMinutes = 15; - await packageCache.set(cacheNamespace, cacheKey, dep, cacheMinutes); - return dep; - } catch (err) /* istanbul ignore next */ { - logger.debug({ err }, 'CircleCI Orb lookup error'); - if (err.statusCode === 404 || err.code === 'ENOTFOUND') { - logger.debug({ lookupName }, `CircleCI Orb lookup failure: not found`); - return null; - } - logger.warn({ lookupName }, 'CircleCI Orb lookup failure: Unknown error'); + const res: OrbRelease = ( + await http.postJson<{ data: { orb: OrbRelease } }>(url, { + body, + }) + ).body.data.orb; + if (!res) { + logger.debug({ lookupName }, 'Failed to look up orb'); return null; } + // Simplify response before caching and returning + const dep: ReleaseResult = { + name: lookupName, + versions: {}, + releases: null, + }; + if (res.homeUrl && res.homeUrl.length) { + dep.homepage = res.homeUrl; + } + dep.homepage = + dep.homepage || `https://circleci.com/orbs/registry/orb/${lookupName}`; + dep.releases = res.versions.map(({ version, createdAt }) => ({ + version, + releaseTimestamp: createdAt || null, + })); + logger.trace({ dep }, 'dep'); + const cacheMinutes = 15; + await packageCache.set(cacheNamespace, cacheKey, dep, cacheMinutes); + return dep; } diff --git a/lib/datasource/packagist/index.ts b/lib/datasource/packagist/index.ts index 78b8d175dbe5c4..ba425b8c8034bd 100644 --- a/lib/datasource/packagist/index.ts +++ b/lib/datasource/packagist/index.ts @@ -49,63 +49,42 @@ interface RegistryMeta { } async function getRegistryMeta(regUrl: string): Promise { - try { - const url = URL.resolve(regUrl.replace(/\/?$/, '/'), 'packages.json'); - const opts = getHostOpts(url); - const res = (await http.getJson(url, opts)).body; - const meta: RegistryMeta = { - providerPackages: {}, - }; - meta.packages = res.packages; - if (res.includes) { - meta.includesFiles = []; - for (const [name, val] of Object.entries(res.includes)) { - const file = { - key: name.replace(val.sha256, '%hash%'), - sha256: val.sha256, - }; - meta.includesFiles.push(file); - } - } - if (res['providers-url']) { - meta.providersUrl = res['providers-url']; - } - if (res['provider-includes']) { - meta.files = []; - for (const [key, val] of Object.entries(res['provider-includes'])) { - const file = { - key, - sha256: val.sha256, - }; - meta.files.push(file); - } - } - if (res.providers) { - for (const [key, val] of Object.entries(res.providers)) { - meta.providerPackages[key] = val.sha256; - } - } - return meta; - } catch (err) { - if (err.code === 'ETIMEDOUT') { - logger.debug({ regUrl }, 'Packagist timeout'); - return null; + const url = URL.resolve(regUrl.replace(/\/?$/, '/'), 'packages.json'); + const opts = getHostOpts(url); + const res = (await http.getJson(url, opts)).body; + const meta: RegistryMeta = { + providerPackages: {}, + }; + meta.packages = res.packages; + if (res.includes) { + meta.includesFiles = []; + for (const [name, val] of Object.entries(res.includes)) { + const file = { + key: name.replace(val.sha256, '%hash%'), + sha256: val.sha256, + }; + meta.includesFiles.push(file); } - if (err.statusCode === 401 || err.statusCode === 403) { - logger.debug({ regUrl }, 'Unauthorized Packagist repository'); - return null; + } + if (res['providers-url']) { + meta.providersUrl = res['providers-url']; + } + if (res['provider-includes']) { + meta.files = []; + for (const [key, val] of Object.entries(res['provider-includes'])) { + const file = { + key, + sha256: val.sha256, + }; + meta.files.push(file); } - if ( - err.statusCode === 404 && - err.url && - err.url.endsWith('/packages.json') - ) { - logger.debug({ regUrl }, 'Packagist repository not found'); - return null; + } + if (res.providers) { + for (const [key, val] of Object.entries(res.providers)) { + meta.providerPackages[key] = val.sha256; } - logger.warn({ err }, 'Packagist download error'); - return null; } + return meta; } interface PackagistFile { @@ -176,9 +155,6 @@ interface AllPackages { async function getAllPackages(regUrl: string): Promise { const registryMeta = await getRegistryMeta(regUrl); - if (!registryMeta) { - return null; - } const { packages, providersUrl, @@ -266,9 +242,6 @@ async function packageLookup( return packagistResult; } const allPackages = await getAllCachedPackages(regUrl); - if (!allPackages) { - return null; - } const { packages, providersUrl, @@ -302,16 +275,6 @@ async function packageLookup( logger.trace({ dep }, 'dep'); return dep; } catch (err) /* istanbul ignore next */ { - if (err.statusCode === 404 || err.code === 'ENOTFOUND') { - logger.debug( - { dependency: name }, - `Dependency lookup failure: not found` - ); - logger.debug({ - err, - }); - return null; - } if (err.host === 'packagist.org') { if (err.code === 'ECONNRESET' || err.code === 'ETIMEDOUT') { throw new ExternalHostError(err); @@ -320,8 +283,7 @@ async function packageLookup( throw new ExternalHostError(err); } } - logger.warn({ err, name }, 'packagist registry failure: Unknown error'); - return null; + throw err; } } diff --git a/lib/datasource/pypi/index.ts b/lib/datasource/pypi/index.ts index a351082324e22c..56c3a9a605a4ba 100644 --- a/lib/datasource/pypi/index.ts +++ b/lib/datasource/pypi/index.ts @@ -58,88 +58,74 @@ async function getDependency( hostUrl: string, compatibility: Record ): Promise { - try { - const lookupUrl = url.resolve(hostUrl, `${packageName}/json`); - const dependency: ReleaseResult = { releases: null }; - logger.trace({ lookupUrl }, 'Pypi api got lookup'); - const rep = await http.getJson(lookupUrl); - const dep = rep && rep.body; - if (!dep) { - logger.trace({ dependency: packageName }, 'pip package not found'); - return null; - } - logger.trace({ lookupUrl }, 'Got pypi api result'); - if ( - !(dep.info && normalizeName(dep.info.name) === normalizeName(packageName)) - ) { - logger.warn( - { lookupUrl, lookupName: packageName, returnedName: dep.info.name }, - 'Returned name does not match with requested name' - ); - return null; + const lookupUrl = url.resolve(hostUrl, `${packageName}/json`); + const dependency: ReleaseResult = { releases: null }; + logger.trace({ lookupUrl }, 'Pypi api got lookup'); + const rep = await http.getJson(lookupUrl); + const dep = rep && rep.body; + if (!dep) { + logger.trace({ dependency: packageName }, 'pip package not found'); + return null; + } + logger.trace({ lookupUrl }, 'Got pypi api result'); + if ( + !(dep.info && normalizeName(dep.info.name) === normalizeName(packageName)) + ) { + logger.warn( + { lookupUrl, lookupName: packageName, returnedName: dep.info.name }, + 'Returned name does not match with requested name' + ); + return null; + } + + if (dep.info?.home_page) { + dependency.homepage = dep.info.home_page; + if (github_repo_pattern.exec(dep.info.home_page)) { + dependency.sourceUrl = dep.info.home_page.replace('http://', 'https://'); } + } - if (dep.info?.home_page) { - dependency.homepage = dep.info.home_page; - if (github_repo_pattern.exec(dep.info.home_page)) { - dependency.sourceUrl = dep.info.home_page.replace( - 'http://', - 'https://' - ); + if (dep.info?.project_urls) { + for (const [name, projectUrl] of Object.entries(dep.info.project_urls)) { + const lower = name.toLowerCase(); + + if ( + !dependency.sourceUrl && + (lower.startsWith('repo') || + lower === 'code' || + lower === 'source' || + github_repo_pattern.exec(projectUrl)) + ) { + dependency.sourceUrl = projectUrl; } - } - if (dep.info?.project_urls) { - for (const [name, projectUrl] of Object.entries(dep.info.project_urls)) { - const lower = name.toLowerCase(); - - if ( - !dependency.sourceUrl && - (lower.startsWith('repo') || - lower === 'code' || - lower === 'source' || - github_repo_pattern.exec(projectUrl)) - ) { - dependency.sourceUrl = projectUrl; - } - - if ( - !dependency.changelogUrl && - ([ - 'changelog', - 'change log', - 'changes', - 'release notes', - 'news', - "what's new", - ].includes(lower) || - changelogFilenameRegex.exec(lower)) - ) { - // from https://github.com/pypa/warehouse/blob/418c7511dc367fb410c71be139545d0134ccb0df/warehouse/templates/packaging/detail.html#L24 - dependency.changelogUrl = projectUrl; - } + if ( + !dependency.changelogUrl && + ([ + 'changelog', + 'change log', + 'changes', + 'release notes', + 'news', + "what's new", + ].includes(lower) || + changelogFilenameRegex.exec(lower)) + ) { + // from https://github.com/pypa/warehouse/blob/418c7511dc367fb410c71be139545d0134ccb0df/warehouse/templates/packaging/detail.html#L24 + dependency.changelogUrl = projectUrl; } } + } - dependency.releases = []; - if (dep.releases) { - const versions = compatibleVersions(dep.releases, compatibility); - dependency.releases = versions.map((version) => ({ - version, - releaseTimestamp: (dep.releases[version][0] || {}).upload_time, - })); - } - return dependency; - } catch (err) { - logger.debug( - 'pypi dependency not found: ' + - packageName + - '(searching in ' + - hostUrl + - ')' - ); - return null; + dependency.releases = []; + if (dep.releases) { + const versions = compatibleVersions(dep.releases, compatibility); + dependency.releases = versions.map((version) => ({ + version, + releaseTimestamp: (dep.releases[version][0] || {}).upload_time, + })); } + return dependency; } function extractVersionFromLinkText( @@ -174,40 +160,29 @@ async function getSimpleDependency( hostUrl: string ): Promise { const lookupUrl = url.resolve(hostUrl, `${packageName}`); - try { - const dependency: ReleaseResult = { releases: null }; - const response = await http.get(lookupUrl); - const dep = response && response.body; - if (!dep) { - logger.trace({ dependency: packageName }, 'pip package not found'); - return null; - } - const root: HTMLElement = parse(dep.replace(/<\/?pre>/, '')) as any; - const links = root.querySelectorAll('a'); - const versions = new Set(); - for (const link of Array.from(links)) { - const result = extractVersionFromLinkText(link.text, packageName); - if (result) { - versions.add(result); - } - } - dependency.releases = []; - if (versions && versions.size > 0) { - dependency.releases = [...versions].map((version) => ({ - version, - })); - } - return dependency; - } catch (err) { - logger.debug( - 'pypi dependency not found: ' + - packageName + - '(searching in ' + - hostUrl + - ')' - ); + const dependency: ReleaseResult = { releases: null }; + const response = await http.get(lookupUrl); + const dep = response && response.body; + if (!dep) { + logger.trace({ dependency: packageName }, 'pip package not found'); return null; } + const root: HTMLElement = parse(dep.replace(/<\/?pre>/, '')) as any; + const links = root.querySelectorAll('a'); + const versions = new Set(); + for (const link of Array.from(links)) { + const result = extractVersionFromLinkText(link.text, packageName); + if (result) { + versions.add(result); + } + } + dependency.releases = []; + if (versions && versions.size > 0) { + dependency.releases = [...versions].map((version) => ({ + version, + })); + } + return dependency; } export async function getReleases({ diff --git a/lib/datasource/rubygems/errors.ts b/lib/datasource/rubygems/errors.ts deleted file mode 100644 index 7c9b7971d88e09..00000000000000 --- a/lib/datasource/rubygems/errors.ts +++ /dev/null @@ -1,17 +0,0 @@ -const UNAUTHORIZED = 401; -const FORBIDDEN = 403; - -const NOT_FOUND = 404; - -const REQUEST_TIMEOUT = 408; -const TOO_MANY_REQUEST = 429; -const SERVICE_UNAVAILABLE = 503; - -export { - UNAUTHORIZED, - FORBIDDEN, - NOT_FOUND, - REQUEST_TIMEOUT, - TOO_MANY_REQUEST, - SERVICE_UNAVAILABLE, -}; diff --git a/lib/datasource/rubygems/get.ts b/lib/datasource/rubygems/get.ts index 91e72767f0d136..55e1d4d2f0e51f 100644 --- a/lib/datasource/rubygems/get.ts +++ b/lib/datasource/rubygems/get.ts @@ -1,37 +1,15 @@ import { OutgoingHttpHeaders } from 'http'; import { logger } from '../../logger'; import { Http } from '../../util/http'; -import { maskToken } from '../../util/mask'; import { ensureTrailingSlash } from '../../util/url'; import { ReleaseResult } from '../common'; import { id } from './common'; -import { FORBIDDEN, NOT_FOUND, UNAUTHORIZED } from './errors'; const http = new Http(id); const INFO_PATH = '/api/v1/gems'; const VERSIONS_PATH = '/api/v1/versions'; -// istanbul ignore next -const processError = ({ err, ...rest }): null => { - const { code, statusCode, headers = {} } = err; - const data = { - ...rest, - err, - statusCode, - token: maskToken(headers.authorization) || 'none', - }; - - if (code === 'ENOTFOUND' || statusCode === NOT_FOUND) { - logger.debug(data, 'RubyGems lookup failure: not found'); - } else if (statusCode === FORBIDDEN || statusCode === UNAUTHORIZED) { - logger.debug(data, 'RubyGems lookup failure: authentication failed'); - } else { - logger.debug(data, 'RubyGems lookup failure: unknown reason'); - } - return null; -}; - const getHeaders = (): OutgoingHttpHeaders => { return { hostType: id }; }; @@ -55,47 +33,43 @@ export const getDependency = async ({ registry, }): Promise => { logger.debug({ dependency }, 'RubyGems lookup for dependency'); - try { - const info = await fetch({ dependency, registry, path: INFO_PATH }); - if (!info) { - logger.debug({ dependency }, 'RubyGems package not found.'); - return null; - } + const info = await fetch({ dependency, registry, path: INFO_PATH }); + if (!info) { + logger.debug({ dependency }, 'RubyGems package not found.'); + return null; + } - if (dependency.toLowerCase() !== info.name.toLowerCase()) { - logger.warn( - { lookup: dependency, returned: info.name }, - 'Lookup name does not match with returned.' - ); - return null; - } + if (dependency.toLowerCase() !== info.name.toLowerCase()) { + logger.warn( + { lookup: dependency, returned: info.name }, + 'Lookup name does not match with returned.' + ); + return null; + } - const versions = - (await fetch({ dependency, registry, path: VERSIONS_PATH })) || []; + const versions = + (await fetch({ dependency, registry, path: VERSIONS_PATH })) || []; - const releases = versions.map( - ({ - number: version, - platform: rubyPlatform, - created_at: releaseTimestamp, - rubygems_version: rubygemsVersion, - ruby_version: rubyVersion, - }) => ({ - version, - rubyPlatform, - releaseTimestamp, - rubygemsVersion, - rubyVersion, - }) - ); + const releases = versions.map( + ({ + number: version, + platform: rubyPlatform, + created_at: releaseTimestamp, + rubygems_version: rubygemsVersion, + ruby_version: rubyVersion, + }) => ({ + version, + rubyPlatform, + releaseTimestamp, + rubygemsVersion, + rubyVersion, + }) + ); - return { - releases, - homepage: info.homepage_uri, - sourceUrl: info.source_code_uri, - changelogUrl: info.changelog_uri, - }; - } catch (err) { - return processError({ err, registry, dependency }); - } + return { + releases, + homepage: info.homepage_uri, + sourceUrl: info.source_code_uri, + changelogUrl: info.changelog_uri, + }; }; diff --git a/lib/datasource/terraform-module/index.ts b/lib/datasource/terraform-module/index.ts index 217cdcb9827294..50c7a7b90d016e 100644 --- a/lib/datasource/terraform-module/index.ts +++ b/lib/datasource/terraform-module/index.ts @@ -101,25 +101,11 @@ export async function getReleases({ await packageCache.set(cacheNamespace, pkgUrl, dep, cacheMinutes); return dep; } catch (err) { - if (err.statusCode === 404 || err.code === 'ENOTFOUND') { - logger.debug( - { lookupName }, - `Terraform registry lookup failure: not found` - ); - logger.debug({ - err, - }); - return null; - } const failureCodes = ['EAI_AGAIN']; // istanbul ignore if if (failureCodes.includes(err.code)) { throw new ExternalHostError(err); } - logger.warn( - { err, lookupName }, - 'Terraform registry failure: Unknown error' - ); - return null; + throw err; } } diff --git a/lib/datasource/terraform-provider/index.ts b/lib/datasource/terraform-provider/index.ts index 5ba096a98e3388..64be934edfee1b 100644 --- a/lib/datasource/terraform-provider/index.ts +++ b/lib/datasource/terraform-provider/index.ts @@ -36,33 +36,22 @@ async function queryRegistry( registryURL: string, repository: string ): Promise { - try { - const backendURL = `${registryURL}/v1/providers/${repository}`; - const res = (await http.getJson(backendURL)).body; - const dep: ReleaseResult = { - name: repository, - versions: {}, - releases: null, - }; - if (res.source) { - dep.sourceUrl = res.source; - } - dep.releases = res.versions.map((version) => ({ - version, - })); - dep.homepage = `${registryURL}/providers/${repository}`; - logger.trace({ dep }, 'dep'); - return dep; - } catch (err) { - logger.debug( - { lookupName }, - `Terraform registry ("${registryURL}") lookup failure: not found` - ); - logger.debug({ - err, - }); - return null; + const backendURL = `${registryURL}/v1/providers/${repository}`; + const res = (await http.getJson(backendURL)).body; + const dep: ReleaseResult = { + name: repository, + versions: {}, + releases: null, + }; + if (res.source) { + dep.sourceUrl = res.source; } + dep.releases = res.versions.map((version) => ({ + version, + })); + dep.homepage = `${registryURL}/providers/${repository}`; + logger.trace({ dep }, 'dep'); + return dep; } async function queryReleaseBackend( @@ -72,32 +61,20 @@ async function queryReleaseBackend( ): Promise { const backendLookUpName = `terraform-provider-${lookupName}`; const backendURL = registryURL + `/index.json`; - try { - const res = ( - await http.getJson(backendURL) - ).body; - const dep: ReleaseResult = { - name: repository, - versions: {}, - releases: null, - }; - dep.releases = Object.keys(res[backendLookUpName].versions).map( - (version) => ({ - version, - }) - ); - logger.trace({ dep }, 'dep'); - return dep; - } catch (err) { - logger.debug( - { lookupName }, - `Terraform registry ("${registryURL}") lookup failure: not found` - ); - logger.debug({ - err, - }); - return null; - } + const res = (await http.getJson(backendURL)) + .body; + const dep: ReleaseResult = { + name: repository, + versions: {}, + releases: null, + }; + dep.releases = Object.keys(res[backendLookUpName].versions).map( + (version) => ({ + version, + }) + ); + logger.trace({ dep }, 'dep'); + return dep; } /** From b58b90dc57d9099527faf7526edf7f2eefa3c195 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Thu, 25 Jun 2020 16:37:11 +0200 Subject: [PATCH 22/25] fix(gitlabci): avoid capturing quotation marks in currentValue (#6584) Closes #6516 --- lib/manager/gitlabci/__fixtures__/gitlab-ci.yaml | 2 ++ lib/manager/gitlabci/__snapshots__/extract.spec.ts.snap | 9 +++++++++ lib/manager/gitlabci/extract.spec.ts | 5 ++++- lib/manager/gitlabci/extract.ts | 4 ++-- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/lib/manager/gitlabci/__fixtures__/gitlab-ci.yaml b/lib/manager/gitlabci/__fixtures__/gitlab-ci.yaml index 145811e6aedf8a..e385b37942b5bf 100644 --- a/lib/manager/gitlabci/__fixtures__/gitlab-ci.yaml +++ b/lib/manager/gitlabci/__fixtures__/gitlab-ci.yaml @@ -1,3 +1,5 @@ +image: 'ruby:2.5.0' + .executor-docker: &executor-docker tags: - docker diff --git a/lib/manager/gitlabci/__snapshots__/extract.spec.ts.snap b/lib/manager/gitlabci/__snapshots__/extract.spec.ts.snap index 1587d27e655cf0..cf7c455c59d31d 100644 --- a/lib/manager/gitlabci/__snapshots__/extract.spec.ts.snap +++ b/lib/manager/gitlabci/__snapshots__/extract.spec.ts.snap @@ -2,6 +2,15 @@ exports[`lib/manager/gitlabci/extract extractPackageFile() extracts multiple image lines 1`] = ` Array [ + Object { + "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", + "currentDigest": undefined, + "currentValue": "2.5.0", + "datasource": "docker", + "depName": "ruby", + "depType": "image", + "replaceString": "ruby:2.5.0", + }, Object { "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", "currentDigest": undefined, diff --git a/lib/manager/gitlabci/extract.spec.ts b/lib/manager/gitlabci/extract.spec.ts index 60e8a17ac342f6..269e4d30e44be1 100644 --- a/lib/manager/gitlabci/extract.spec.ts +++ b/lib/manager/gitlabci/extract.spec.ts @@ -19,7 +19,10 @@ describe('lib/manager/gitlabci/extract', () => { it('extracts multiple image lines', () => { const res = extractPackageFile(yamlFile); expect(res.deps).toMatchSnapshot(); - expect(res.deps).toHaveLength(6); + expect(res.deps).toHaveLength(7); + expect(res.deps.some((dep) => dep.currentValue.includes("'"))).toBe( + false + ); }); it('extracts multiple image lines with comments', () => { diff --git a/lib/manager/gitlabci/extract.ts b/lib/manager/gitlabci/extract.ts index 81b4ed8b018838..b3361db186db30 100644 --- a/lib/manager/gitlabci/extract.ts +++ b/lib/manager/gitlabci/extract.ts @@ -20,12 +20,12 @@ export function extractPackageFile(content: string): PackageFile | null { const lines = content.split('\n'); for (let lineNumber = 0; lineNumber < lines.length; lineNumber += 1) { const line = lines[lineNumber]; - const imageMatch = /^\s*image:\s*'?"?([^\s]+|)'?"?\s*$/.exec(line); + const imageMatch = /^\s*image:\s*'?"?([^\s'"]+|)'?"?\s*$/.exec(line); if (imageMatch) { switch (imageMatch[1]) { case '': { const imageNameLine = skipCommentLines(lines, lineNumber + 1); - const imageNameMatch = /^\s*name:\s*'?"?([^\s]+|)'?"?\s*$/.exec( + const imageNameMatch = /^\s*name:\s*'?"?([^\s'"]+|)'?"?\s*$/.exec( imageNameLine.line ); From 0c06a23a3f5130299a564aea7909c7c2d8cae804 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Thu, 25 Jun 2020 16:40:19 +0200 Subject: [PATCH 23/25] fix(hex): better handling of double equals (#6586) Co-authored-by: proton <25139420+proton-ab@users.noreply.github.com> --- lib/versioning/hex/index.spec.ts | 43 +++++++++++++++++++++++++++++--- lib/versioning/hex/index.ts | 7 ++++-- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/lib/versioning/hex/index.spec.ts b/lib/versioning/hex/index.spec.ts index 8894bf1dc6545c..59fc6e8f82b828 100644 --- a/lib/versioning/hex/index.spec.ts +++ b/lib/versioning/hex/index.spec.ts @@ -7,6 +7,8 @@ describe('lib/versioning/hex', () => { expect(hexScheme.matches('2.1.0', '~> 2.0.0')).toBe(false); expect(hexScheme.matches('2.0.0', '>= 2.0.0 and < 2.1.0')).toBe(true); expect(hexScheme.matches('2.1.0', '== 2.0.0 or < 2.1.0')).toBe(false); + expect(hexScheme.matches('1.9.4', '== 1.9.4')).toBe(true); + expect(hexScheme.matches('1.9.5', '== 1.9.4')).toBe(false); }); }); it('handles tilde greater than', () => { @@ -33,6 +35,9 @@ describe('lib/versioning/hex', () => { it('handles !=', () => { expect(hexScheme.isValid('!= 1.0.0')).toBeTruthy(); }); + it('handles ==', () => { + expect(hexScheme.isValid('== 1.0.0')).toBeTruthy(); + }); }); describe('hexScheme.isLessThanRange()', () => { it('handles and', () => { @@ -69,6 +74,36 @@ describe('lib/versioning/hex', () => { }); }); describe('hexScheme.getNewValue()', () => { + it('handles exact pin', () => { + expect( + hexScheme.getNewValue({ + currentValue: '== 1.2.3', + rangeStrategy: 'pin', + fromVersion: '1.2.3', + toVersion: '2.0.7', + }) + ).toEqual('== 2.0.7'); + }); + it('handles exact bump', () => { + expect( + hexScheme.getNewValue({ + currentValue: '== 3.6.1', + rangeStrategy: 'bump', + fromVersion: '3.6.1', + toVersion: '3.6.2', + }) + ).toEqual('== 3.6.2'); + }); + it('handles exact replace', () => { + expect( + hexScheme.getNewValue({ + currentValue: '== 3.6.1', + rangeStrategy: 'replace', + fromVersion: '3.6.1', + toVersion: '3.6.2', + }) + ).toEqual('== 3.6.2'); + }); it('handles tilde greater than', () => { expect( hexScheme.getNewValue({ @@ -85,7 +120,7 @@ describe('lib/versioning/hex', () => { fromVersion: '1.2.3', toVersion: '2.0.7', }) - ).toEqual('2.0.7'); + ).toEqual('== 2.0.7'); expect( hexScheme.getNewValue({ currentValue: '~> 1.2', @@ -109,7 +144,7 @@ describe('lib/versioning/hex', () => { fromVersion: '1.2.3', toVersion: '2.0.7', }) - ).toEqual('2.0.7'); + ).toEqual('== 2.0.7'); expect( hexScheme.getNewValue({ currentValue: '~> 1.2.0', @@ -144,7 +179,7 @@ describe('lib/versioning/hex', () => { fromVersion: '1.2.3', toVersion: '2.0.7', }) - ).toEqual('2.0.7'); + ).toEqual('== 2.0.7'); }); it('handles or', () => { expect( @@ -170,6 +205,6 @@ describe('lib/versioning/hex', () => { fromVersion: '1.2.3', toVersion: '2.0.7', }) - ).toEqual('2.0.7'); + ).toEqual('== 2.0.7'); }); }); diff --git a/lib/versioning/hex/index.ts b/lib/versioning/hex/index.ts index 5608a58115a49e..47ab7afa0b180b 100644 --- a/lib/versioning/hex/index.ts +++ b/lib/versioning/hex/index.ts @@ -13,7 +13,8 @@ function hex2npm(input: string): string { .replace(/~>\s*(\d+\.\d+\.\d+)/, '~$1') .replace(/==|and/, '') .replace('or', '||') - .replace(/!=\s*(\d+\.\d+(\.\d+.*)?)/, '>$1 <$1'); + .replace(/!=\s*(\d+\.\d+(\.\d+.*)?)/, '>$1 <$1') + .trim(); } function npm2hex(input: string): string { @@ -76,7 +77,9 @@ const getNewValue = ({ } else { newSemver = newSemver.replace(/~\s*(\d+\.\d+\.\d)/, '~> $1'); } - + if (npm.isVersion(newSemver)) { + newSemver = `== ${newSemver}`; + } return newSemver; }; From 7f5ce4336ab78f680123f68621238b1c2cc90fa6 Mon Sep 17 00:00:00 2001 From: Sergio Zharinov Date: Fri, 26 Jun 2020 00:53:18 +0400 Subject: [PATCH 24/25] fix(poetry): Fix range unions (#6587) --- lib/versioning/poetry/index.spec.ts | 10 ++++++++++ lib/versioning/poetry/index.ts | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/versioning/poetry/index.spec.ts b/lib/versioning/poetry/index.spec.ts index b6d0f96a6091ec..94862bfad48da9 100644 --- a/lib/versioning/poetry/index.spec.ts +++ b/lib/versioning/poetry/index.spec.ts @@ -457,4 +457,14 @@ describe('semver.getNewValue()', () => { }) ).toEqual('~2'); }); + it('widens range', () => { + expect( + semver.getNewValue({ + currentValue: '^2.2', + rangeStrategy: 'widen', + fromVersion: '2.2.0', + toVersion: '3.0.0', + }) + ).toEqual('^2.2 || ^3.0.0'); + }); }); diff --git a/lib/versioning/poetry/index.ts b/lib/versioning/poetry/index.ts index 898184883ec7cc..19adea10d45a12 100644 --- a/lib/versioning/poetry/index.ts +++ b/lib/versioning/poetry/index.ts @@ -40,7 +40,7 @@ function npm2poetry(input: string): string { res.splice(i, 2, newValue); } } - return res.join(', '); + return res.join(', ').replace(/\s*,?\s*\|\|\s*,?\s*/, ' || '); } const isLessThanRange = (version: string, range: string): boolean => From af203f6f90fdba63975a078790521a5dd05dbfb4 Mon Sep 17 00:00:00 2001 From: Wolfgang Faust Date: Thu, 25 Jun 2020 21:36:35 -0700 Subject: [PATCH 25/25] docs: fix coverage report location (#6590) --- docs/development/local-development.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/development/local-development.md b/docs/development/local-development.md index f938236c1c93ab..b00d6540cebe37 100644 --- a/docs/development/local-development.md +++ b/docs/development/local-development.md @@ -142,7 +142,7 @@ You also need to make sure that you don't have a local `.npmrc` file that overri The Renovate project maintains 100% test coverage, so any Pull Request will fail if it does not contain full coverage for code. Using `// istanbul ignore` is not ideal but sometimes is a pragmatic solution if an additional test wouldn't really prove anything. -To view the current test coverage locally, open up `coverage/lcov-report/index.html` in your browser. +To view the current test coverage locally, open up `coverage/index.html` in your browser. Do not let coverage put you off submitting a PR! Maybe we can help, or at least guide. Also, it can be good to submit your PR as a work in progress (WIP) without tests first so that you can get a thumbs up from others about the changes, and write tests after.