diff --git a/lib/datasource/cdnjs/index.ts b/lib/datasource/cdnjs/index.ts index a994a6b8c639e8..58f912c6454202 100644 --- a/lib/datasource/cdnjs/index.ts +++ b/lib/datasource/cdnjs/index.ts @@ -1,4 +1,3 @@ -import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; import { Http } from '../../util/http'; import { CachePromise, cacheAble } from '../cache'; @@ -57,11 +56,9 @@ export async function getReleases({ } return result; } catch (err) { - if (err.statusCode === 404) { - logger.debug({ library }, 'cdnjs library not found'); - return null; + if (err.statusCode !== 404) { + throw new ExternalHostError(err); } - // Throw an ExternalHostError for all other types of errors - throw new ExternalHostError(err); + throw err; } } diff --git a/lib/datasource/crate/index.ts b/lib/datasource/crate/index.ts index 9715b4eab18353..324dad6e0065f8 100644 --- a/lib/datasource/crate/index.ts +++ b/lib/datasource/crate/index.ts @@ -1,4 +1,3 @@ -import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; import * as packageCache from '../../util/cache/package'; import { Http } from '../../util/http'; @@ -66,18 +65,12 @@ export async function getReleases({ await packageCache.set(cacheNamespace, cacheKey, result, cacheMinutes); return result; } catch (err) { - if (err.statusCode === 404 || err.code === 'ENOTFOUND') { - logger.debug({ lookupName }, `Dependency lookup failure: not found`); - logger.debug({ err }, 'Crate lookup error'); - return null; - } if ( err.statusCode === 429 || (err.statusCode >= 500 && err.statusCode < 600) ) { throw new ExternalHostError(err); } - logger.warn({ err, lookupName }, 'crates.io lookup failure: Unknown error'); - return null; + throw err; } } diff --git a/lib/datasource/dart/index.ts b/lib/datasource/dart/index.ts index 241394373b5d7b..94f6dc4d218ae0 100644 --- a/lib/datasource/dart/index.ts +++ b/lib/datasource/dart/index.ts @@ -1,4 +1,3 @@ -import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; import { Http, HttpResponse } from '../../util/http'; import { GetReleasesConfig, ReleaseResult } from '../common'; @@ -25,22 +24,13 @@ export async function getReleases({ try { raw = await http.getJson(pkgUrl); } catch (err) { - if (err.statusCode === 404 || err.code === 'ENOTFOUND') { - logger.debug({ lookupName }, `Dependency lookup failure: not found`); - logger.debug({ err }, 'Dart lookup error'); - return null; - } if ( err.statusCode === 429 || (err.statusCode >= 500 && err.statusCode < 600) ) { throw new ExternalHostError(err); } - logger.warn( - { err, lookupName }, - 'pub.dartlang.org lookup failure: Unknown error' - ); - return null; + throw err; } const body = raw && raw.body; diff --git a/lib/datasource/docker/index.ts b/lib/datasource/docker/index.ts index 4af0da52eac34e..bb7d5744834710 100644 --- a/lib/datasource/docker/index.ts +++ b/lib/datasource/docker/index.ts @@ -417,25 +417,12 @@ async function getTags( if (err instanceof ExternalHostError) { throw err; } - logger.debug( - { - err, - }, - 'docker.getTags() error' - ); if (err.statusCode === 404 && !repository.includes('/')) { logger.debug( `Retrying Tags for ${registry}/${repository} using library/ prefix` ); return getTags(registry, 'library/' + repository); } - if (err.statusCode === 401 || err.statusCode === 403) { - logger.debug( - { registry, dockerRepository: repository, err }, - 'Not authorised to look up docker tags' - ); - return null; - } // prettier-ignore if (err.statusCode === 429 && registry.endsWith('docker.io')) { // lgtm [js/incomplete-url-substring-sanitization] logger.warn( @@ -451,18 +438,7 @@ async function getTags( ); throw new ExternalHostError(err); } - if (err.code === 'ETIMEDOUT') { - logger.debug( - { registry }, - 'Timeout when attempting to connect to docker registry' - ); - return null; - } - logger.warn( - { registry, dockerRepository: repository, err }, - 'Error getting docker image tags' - ); - return null; + throw err; } } diff --git a/lib/datasource/galaxy/index.ts b/lib/datasource/galaxy/index.ts index ed01e05fe8a442..48ef50b7c782df 100644 --- a/lib/datasource/galaxy/index.ts +++ b/lib/datasource/galaxy/index.ts @@ -99,6 +99,6 @@ export async function getReleases({ ) { throw new ExternalHostError(err); } - return null; + throw err; } } diff --git a/lib/datasource/git-refs/index.ts b/lib/datasource/git-refs/index.ts index 50820ee5920950..c872bc33f23336 100644 --- a/lib/datasource/git-refs/index.ts +++ b/lib/datasource/git-refs/index.ts @@ -1,5 +1,4 @@ import simpleGit from 'simple-git/promise'; -import { logger } from '../../logger'; import * as packageCache from '../../util/cache/package'; import * as semver from '../../versioning/semver'; import { DigestConfig, GetReleasesConfig, ReleaseResult } from '../common'; @@ -21,90 +20,80 @@ export async function getRawRefs({ lookupName, }: GetReleasesConfig): Promise { const git = simpleGit(); - try { - const cacheNamespace = 'git-raw-refs'; - - const cachedResult = await packageCache.get( - cacheNamespace, - lookupName - ); - /* istanbul ignore next line */ - if (cachedResult) { - return cachedResult; - } - - // fetch remote tags - const lsRemote = await git.listRemote([lookupName]); - if (!lsRemote) { - return null; - } - - const refMatch = /(?.*?)\s+refs\/(?.*?)\/(?.*)/; - const headMatch = /(?.*?)\s+HEAD/; - - const refs = lsRemote - .trim() - .split('\n') - .map((line) => line.trim()) - .map((line) => { - let match = refMatch.exec(line); - if (match) { - return { - type: match.groups.type, - value: match.groups.value, - hash: match.groups.hash, - }; - } - match = headMatch.exec(line); - if (match) { - return { - type: '', - value: 'HEAD', - hash: match.groups.hash, - }; - } - // istanbul ignore next - return null; - }) - .filter(Boolean) - .filter((ref) => ref.type !== 'pull' && !ref.value.endsWith('^{}')); - await packageCache.set(cacheNamespace, lookupName, refs, cacheMinutes); - return refs; - } catch (err) { - logger.info({ err }, `Git-Raw-Refs lookup error in ${lookupName}`); + const cacheNamespace = 'git-raw-refs'; + + const cachedResult = await packageCache.get( + cacheNamespace, + lookupName + ); + /* istanbul ignore next line */ + if (cachedResult) { + return cachedResult; } - return null; + + // fetch remote tags + const lsRemote = await git.listRemote([lookupName]); + if (!lsRemote) { + return null; + } + + const refMatch = /(?.*?)\s+refs\/(?.*?)\/(?.*)/; + const headMatch = /(?.*?)\s+HEAD/; + + const refs = lsRemote + .trim() + .split('\n') + .map((line) => line.trim()) + .map((line) => { + let match = refMatch.exec(line); + if (match) { + return { + type: match.groups.type, + value: match.groups.value, + hash: match.groups.hash, + }; + } + match = headMatch.exec(line); + if (match) { + return { + type: '', + value: 'HEAD', + hash: match.groups.hash, + }; + } + // istanbul ignore next + return null; + }) + .filter(Boolean) + .filter((ref) => ref.type !== 'pull' && !ref.value.endsWith('^{}')); + await packageCache.set(cacheNamespace, lookupName, refs, cacheMinutes); + return refs; } export async function getReleases({ lookupName, }: GetReleasesConfig): Promise { - try { - const rawRefs: RawRefs[] = await getRawRefs({ lookupName }); - - const refs = rawRefs - .filter((ref) => ref.type === 'tags' || ref.type === 'heads') - .map((ref) => ref.value) - .filter((ref) => semver.isVersion(ref)); - - const uniqueRefs = [...new Set(refs)]; - - const sourceUrl = lookupName.replace(/\.git$/, '').replace(/\/$/, ''); - - const result: ReleaseResult = { - sourceUrl, - releases: uniqueRefs.map((ref) => ({ - version: ref, - gitRef: ref, - newDigest: rawRefs.find((rawRef) => rawRef.value === ref).hash, - })), - }; - - return result; - } catch (err) { - logger.error({ err }, `Git-Refs lookup error in ${lookupName}`); - } - return null; + const rawRefs: RawRefs[] = await getRawRefs({ lookupName }); + + const refs = rawRefs + .filter((ref) => ref.type === 'tags' || ref.type === 'heads') + .map((ref) => ref.value) + .filter((ref) => semver.isVersion(ref)); + + const uniqueRefs = [...new Set(refs)]; + + const sourceUrl = lookupName.replace(/\.git$/, '').replace(/\/$/, ''); + + const result: ReleaseResult = { + sourceUrl, + releases: uniqueRefs.map((ref) => ({ + version: ref, + gitRef: ref, + newDigest: rawRefs.find((rawRef) => rawRef.value === ref).hash, + })), + }; + + return result; } export async function getDigest( diff --git a/lib/datasource/git-submodules/index.ts b/lib/datasource/git-submodules/index.ts index 4a812a3fb748f9..22a8b6adeab013 100644 --- a/lib/datasource/git-submodules/index.ts +++ b/lib/datasource/git-submodules/index.ts @@ -1,7 +1,6 @@ import { URL } from 'url'; import Git from 'simple-git/promise'; -import { logger } from '../../logger'; import * as packageCache from '../../util/cache/package'; import { DigestConfig, GetReleasesConfig, ReleaseResult } from '../common'; @@ -23,31 +22,26 @@ export async function getReleases({ } const git = Git(); - try { - const newHash = ( - await git.listRemote(['--refs', registryUrls[0], registryUrls[1]]) - ) - .trim() - .split(/\t/)[0]; + const newHash = ( + await git.listRemote(['--refs', registryUrls[0], registryUrls[1]]) + ) + .trim() + .split(/\t/)[0]; - const sourceUrl = new URL(registryUrls[0]); - sourceUrl.username = ''; + const sourceUrl = new URL(registryUrls[0]); + sourceUrl.username = ''; - const result = { - sourceUrl: sourceUrl.href, - releases: [ - { - version: newHash, - }, - ], - }; - const cacheMinutes = 60; - await packageCache.set(cacheNamespace, cacheKey, result, cacheMinutes); - return result; - } catch (err) { - logger.debug({ err }, `Git-SubModules lookup error in ${lookupName}`); - } - return null; + const result = { + sourceUrl: sourceUrl.href, + releases: [ + { + version: newHash, + }, + ], + }; + const cacheMinutes = 60; + await packageCache.set(cacheNamespace, cacheKey, result, cacheMinutes); + return result; } export const getDigest = ( diff --git a/lib/datasource/github-releases/index.ts b/lib/datasource/github-releases/index.ts index 438f6509ac5f6a..de2ecd435b16a3 100644 --- a/lib/datasource/github-releases/index.ts +++ b/lib/datasource/github-releases/index.ts @@ -1,4 +1,3 @@ -import { logger } from '../../logger'; import * as packageCache from '../../util/cache/package'; import { GithubHttp } from '../../util/http/github'; import { GetReleasesConfig, ReleaseResult } from '../common'; @@ -27,7 +26,6 @@ type GithubRelease = { export async function getReleases({ lookupName: repo, }: GetReleasesConfig): Promise { - let githubReleases: GithubRelease[]; const cachedResult = await packageCache.get( cacheNamespace, repo @@ -36,19 +34,11 @@ export async function getReleases({ if (cachedResult) { return cachedResult; } - try { - const url = `https://api.github.com/repos/${repo}/releases?per_page=100`; - const res = await http.getJson(url, { - paginate: true, - }); - githubReleases = res.body; - } catch (err) /* istanbul ignore next */ { - logger.debug({ repo, err }, 'Error retrieving from github'); - } - // istanbul ignore if - if (!githubReleases) { - return null; - } + const url = `https://api.github.com/repos/${repo}/releases?per_page=100`; + const res = await http.getJson(url, { + paginate: true, + }); + const githubReleases = res.body; const dependency: ReleaseResult = { sourceUrl: 'https://github.com/' + repo, releases: null, diff --git a/lib/datasource/github-tags/index.ts b/lib/datasource/github-tags/index.ts index d2a385a42bafac..2e0877a9f70b10 100644 --- a/lib/datasource/github-tags/index.ts +++ b/lib/datasource/github-tags/index.ts @@ -121,7 +121,6 @@ export async function getDigest( export async function getReleases({ lookupName: repo, }: GetReleasesConfig): Promise { - let versions: string[]; const cachedResult = await packageCache.get( cacheNamespace, getCacheKey(repo, 'tags') @@ -130,24 +129,17 @@ export async function getReleases({ if (cachedResult) { return cachedResult; } - try { - // tag - const url = `https://api.github.com/repos/${repo}/tags?per_page=100`; - type GitHubTag = { - name: string; - }[]; + // tag + const url = `https://api.github.com/repos/${repo}/tags?per_page=100`; + type GitHubTag = { + name: string; + }[]; - versions = ( - await http.getJson(url, { - paginate: true, - }) - ).body.map((o) => o.name); - } catch (err) { - logger.debug({ repo, err }, 'Error retrieving from github'); - } - if (!versions) { - return null; - } + const versions = ( + await http.getJson(url, { + paginate: true, + }) + ).body.map((o) => o.name); const dependency: ReleaseResult = { sourceUrl: 'https://github.com/' + repo, releases: null, diff --git a/lib/datasource/gitlab-tags/index.ts b/lib/datasource/gitlab-tags/index.ts index 59bff4283b83e6..22d481253e7b34 100644 --- a/lib/datasource/gitlab-tags/index.ts +++ b/lib/datasource/gitlab-tags/index.ts @@ -1,5 +1,4 @@ import URL from 'url'; -import { logger } from '../../logger'; import * as packageCache from '../../util/cache/package'; import { GitlabHttp } from '../../util/http/gitlab'; import { GetReleasesConfig, ReleaseResult } from '../common'; @@ -27,7 +26,6 @@ export async function getReleases({ registryUrl: depHost, lookupName: repo, }: GetReleasesConfig): Promise { - let gitlabTags: GitlabTag[]; const cachedResult = await packageCache.get( cacheNamespace, getCacheKey(depHost, repo) @@ -39,27 +37,17 @@ export async function getReleases({ const urlEncodedRepo = encodeURIComponent(repo); - try { - // tag - const url = URL.resolve( - depHost, - `/api/v4/projects/${urlEncodedRepo}/repository/tags?per_page=100` - ); - - gitlabTags = ( - await gitlabApi.getJson(url, { - paginate: true, - }) - ).body; - } catch (err) { - // istanbul ignore next - logger.debug({ repo, err }, 'Error retrieving from Gitlab'); - } + // tag + const url = URL.resolve( + depHost, + `/api/v4/projects/${urlEncodedRepo}/repository/tags?per_page=100` + ); - // istanbul ignore if - if (!gitlabTags) { - return null; - } + const gitlabTags = ( + await gitlabApi.getJson(url, { + paginate: true, + }) + ).body; const dependency: ReleaseResult = { sourceUrl: URL.resolve(depHost, repo), diff --git a/lib/datasource/go/index.ts b/lib/datasource/go/index.ts index 0e658bacf26751..c4411f08dcfeb6 100644 --- a/lib/datasource/go/index.ts +++ b/lib/datasource/go/index.ts @@ -33,40 +33,25 @@ async function getDatasource(goModule: string): Promise { }; } const pkgUrl = `https://${goModule}?go-get=1`; - try { - const res = (await http.get(pkgUrl)).body; - const sourceMatch = regEx( - `(url, { - body, - }) - ).body.data.orb; - if (!res) { - logger.debug({ lookupName }, 'Failed to look up orb'); - return null; - } - // Simplify response before caching and returning - const dep: ReleaseResult = { - name: lookupName, - versions: {}, - releases: null, - }; - if (res.homeUrl && res.homeUrl.length) { - dep.homepage = res.homeUrl; - } - dep.homepage = - dep.homepage || `https://circleci.com/orbs/registry/orb/${lookupName}`; - dep.releases = res.versions.map(({ version, createdAt }) => ({ - version, - releaseTimestamp: createdAt || null, - })); - logger.trace({ dep }, 'dep'); - const cacheMinutes = 15; - await packageCache.set(cacheNamespace, cacheKey, dep, cacheMinutes); - return dep; - } catch (err) /* istanbul ignore next */ { - logger.debug({ err }, 'CircleCI Orb lookup error'); - if (err.statusCode === 404 || err.code === 'ENOTFOUND') { - logger.debug({ lookupName }, `CircleCI Orb lookup failure: not found`); - return null; - } - logger.warn({ lookupName }, 'CircleCI Orb lookup failure: Unknown error'); + const res: OrbRelease = ( + await http.postJson<{ data: { orb: OrbRelease } }>(url, { + body, + }) + ).body.data.orb; + if (!res) { + logger.debug({ lookupName }, 'Failed to look up orb'); return null; } + // Simplify response before caching and returning + const dep: ReleaseResult = { + name: lookupName, + versions: {}, + releases: null, + }; + if (res.homeUrl && res.homeUrl.length) { + dep.homepage = res.homeUrl; + } + dep.homepage = + dep.homepage || `https://circleci.com/orbs/registry/orb/${lookupName}`; + dep.releases = res.versions.map(({ version, createdAt }) => ({ + version, + releaseTimestamp: createdAt || null, + })); + logger.trace({ dep }, 'dep'); + const cacheMinutes = 15; + await packageCache.set(cacheNamespace, cacheKey, dep, cacheMinutes); + return dep; } diff --git a/lib/datasource/packagist/index.ts b/lib/datasource/packagist/index.ts index 78b8d175dbe5c4..ba425b8c8034bd 100644 --- a/lib/datasource/packagist/index.ts +++ b/lib/datasource/packagist/index.ts @@ -49,63 +49,42 @@ interface RegistryMeta { } async function getRegistryMeta(regUrl: string): Promise { - try { - const url = URL.resolve(regUrl.replace(/\/?$/, '/'), 'packages.json'); - const opts = getHostOpts(url); - const res = (await http.getJson(url, opts)).body; - const meta: RegistryMeta = { - providerPackages: {}, - }; - meta.packages = res.packages; - if (res.includes) { - meta.includesFiles = []; - for (const [name, val] of Object.entries(res.includes)) { - const file = { - key: name.replace(val.sha256, '%hash%'), - sha256: val.sha256, - }; - meta.includesFiles.push(file); - } - } - if (res['providers-url']) { - meta.providersUrl = res['providers-url']; - } - if (res['provider-includes']) { - meta.files = []; - for (const [key, val] of Object.entries(res['provider-includes'])) { - const file = { - key, - sha256: val.sha256, - }; - meta.files.push(file); - } - } - if (res.providers) { - for (const [key, val] of Object.entries(res.providers)) { - meta.providerPackages[key] = val.sha256; - } - } - return meta; - } catch (err) { - if (err.code === 'ETIMEDOUT') { - logger.debug({ regUrl }, 'Packagist timeout'); - return null; + const url = URL.resolve(regUrl.replace(/\/?$/, '/'), 'packages.json'); + const opts = getHostOpts(url); + const res = (await http.getJson(url, opts)).body; + const meta: RegistryMeta = { + providerPackages: {}, + }; + meta.packages = res.packages; + if (res.includes) { + meta.includesFiles = []; + for (const [name, val] of Object.entries(res.includes)) { + const file = { + key: name.replace(val.sha256, '%hash%'), + sha256: val.sha256, + }; + meta.includesFiles.push(file); } - if (err.statusCode === 401 || err.statusCode === 403) { - logger.debug({ regUrl }, 'Unauthorized Packagist repository'); - return null; + } + if (res['providers-url']) { + meta.providersUrl = res['providers-url']; + } + if (res['provider-includes']) { + meta.files = []; + for (const [key, val] of Object.entries(res['provider-includes'])) { + const file = { + key, + sha256: val.sha256, + }; + meta.files.push(file); } - if ( - err.statusCode === 404 && - err.url && - err.url.endsWith('/packages.json') - ) { - logger.debug({ regUrl }, 'Packagist repository not found'); - return null; + } + if (res.providers) { + for (const [key, val] of Object.entries(res.providers)) { + meta.providerPackages[key] = val.sha256; } - logger.warn({ err }, 'Packagist download error'); - return null; } + return meta; } interface PackagistFile { @@ -176,9 +155,6 @@ interface AllPackages { async function getAllPackages(regUrl: string): Promise { const registryMeta = await getRegistryMeta(regUrl); - if (!registryMeta) { - return null; - } const { packages, providersUrl, @@ -266,9 +242,6 @@ async function packageLookup( return packagistResult; } const allPackages = await getAllCachedPackages(regUrl); - if (!allPackages) { - return null; - } const { packages, providersUrl, @@ -302,16 +275,6 @@ async function packageLookup( logger.trace({ dep }, 'dep'); return dep; } catch (err) /* istanbul ignore next */ { - if (err.statusCode === 404 || err.code === 'ENOTFOUND') { - logger.debug( - { dependency: name }, - `Dependency lookup failure: not found` - ); - logger.debug({ - err, - }); - return null; - } if (err.host === 'packagist.org') { if (err.code === 'ECONNRESET' || err.code === 'ETIMEDOUT') { throw new ExternalHostError(err); @@ -320,8 +283,7 @@ async function packageLookup( throw new ExternalHostError(err); } } - logger.warn({ err, name }, 'packagist registry failure: Unknown error'); - return null; + throw err; } } diff --git a/lib/datasource/pypi/index.ts b/lib/datasource/pypi/index.ts index a351082324e22c..56c3a9a605a4ba 100644 --- a/lib/datasource/pypi/index.ts +++ b/lib/datasource/pypi/index.ts @@ -58,88 +58,74 @@ async function getDependency( hostUrl: string, compatibility: Record ): Promise { - try { - const lookupUrl = url.resolve(hostUrl, `${packageName}/json`); - const dependency: ReleaseResult = { releases: null }; - logger.trace({ lookupUrl }, 'Pypi api got lookup'); - const rep = await http.getJson(lookupUrl); - const dep = rep && rep.body; - if (!dep) { - logger.trace({ dependency: packageName }, 'pip package not found'); - return null; - } - logger.trace({ lookupUrl }, 'Got pypi api result'); - if ( - !(dep.info && normalizeName(dep.info.name) === normalizeName(packageName)) - ) { - logger.warn( - { lookupUrl, lookupName: packageName, returnedName: dep.info.name }, - 'Returned name does not match with requested name' - ); - return null; + const lookupUrl = url.resolve(hostUrl, `${packageName}/json`); + const dependency: ReleaseResult = { releases: null }; + logger.trace({ lookupUrl }, 'Pypi api got lookup'); + const rep = await http.getJson(lookupUrl); + const dep = rep && rep.body; + if (!dep) { + logger.trace({ dependency: packageName }, 'pip package not found'); + return null; + } + logger.trace({ lookupUrl }, 'Got pypi api result'); + if ( + !(dep.info && normalizeName(dep.info.name) === normalizeName(packageName)) + ) { + logger.warn( + { lookupUrl, lookupName: packageName, returnedName: dep.info.name }, + 'Returned name does not match with requested name' + ); + return null; + } + + if (dep.info?.home_page) { + dependency.homepage = dep.info.home_page; + if (github_repo_pattern.exec(dep.info.home_page)) { + dependency.sourceUrl = dep.info.home_page.replace('http://', 'https://'); } + } - if (dep.info?.home_page) { - dependency.homepage = dep.info.home_page; - if (github_repo_pattern.exec(dep.info.home_page)) { - dependency.sourceUrl = dep.info.home_page.replace( - 'http://', - 'https://' - ); + if (dep.info?.project_urls) { + for (const [name, projectUrl] of Object.entries(dep.info.project_urls)) { + const lower = name.toLowerCase(); + + if ( + !dependency.sourceUrl && + (lower.startsWith('repo') || + lower === 'code' || + lower === 'source' || + github_repo_pattern.exec(projectUrl)) + ) { + dependency.sourceUrl = projectUrl; } - } - if (dep.info?.project_urls) { - for (const [name, projectUrl] of Object.entries(dep.info.project_urls)) { - const lower = name.toLowerCase(); - - if ( - !dependency.sourceUrl && - (lower.startsWith('repo') || - lower === 'code' || - lower === 'source' || - github_repo_pattern.exec(projectUrl)) - ) { - dependency.sourceUrl = projectUrl; - } - - if ( - !dependency.changelogUrl && - ([ - 'changelog', - 'change log', - 'changes', - 'release notes', - 'news', - "what's new", - ].includes(lower) || - changelogFilenameRegex.exec(lower)) - ) { - // from https://github.com/pypa/warehouse/blob/418c7511dc367fb410c71be139545d0134ccb0df/warehouse/templates/packaging/detail.html#L24 - dependency.changelogUrl = projectUrl; - } + if ( + !dependency.changelogUrl && + ([ + 'changelog', + 'change log', + 'changes', + 'release notes', + 'news', + "what's new", + ].includes(lower) || + changelogFilenameRegex.exec(lower)) + ) { + // from https://github.com/pypa/warehouse/blob/418c7511dc367fb410c71be139545d0134ccb0df/warehouse/templates/packaging/detail.html#L24 + dependency.changelogUrl = projectUrl; } } + } - dependency.releases = []; - if (dep.releases) { - const versions = compatibleVersions(dep.releases, compatibility); - dependency.releases = versions.map((version) => ({ - version, - releaseTimestamp: (dep.releases[version][0] || {}).upload_time, - })); - } - return dependency; - } catch (err) { - logger.debug( - 'pypi dependency not found: ' + - packageName + - '(searching in ' + - hostUrl + - ')' - ); - return null; + dependency.releases = []; + if (dep.releases) { + const versions = compatibleVersions(dep.releases, compatibility); + dependency.releases = versions.map((version) => ({ + version, + releaseTimestamp: (dep.releases[version][0] || {}).upload_time, + })); } + return dependency; } function extractVersionFromLinkText( @@ -174,40 +160,29 @@ async function getSimpleDependency( hostUrl: string ): Promise { const lookupUrl = url.resolve(hostUrl, `${packageName}`); - try { - const dependency: ReleaseResult = { releases: null }; - const response = await http.get(lookupUrl); - const dep = response && response.body; - if (!dep) { - logger.trace({ dependency: packageName }, 'pip package not found'); - return null; - } - const root: HTMLElement = parse(dep.replace(/<\/?pre>/, '')) as any; - const links = root.querySelectorAll('a'); - const versions = new Set(); - for (const link of Array.from(links)) { - const result = extractVersionFromLinkText(link.text, packageName); - if (result) { - versions.add(result); - } - } - dependency.releases = []; - if (versions && versions.size > 0) { - dependency.releases = [...versions].map((version) => ({ - version, - })); - } - return dependency; - } catch (err) { - logger.debug( - 'pypi dependency not found: ' + - packageName + - '(searching in ' + - hostUrl + - ')' - ); + const dependency: ReleaseResult = { releases: null }; + const response = await http.get(lookupUrl); + const dep = response && response.body; + if (!dep) { + logger.trace({ dependency: packageName }, 'pip package not found'); return null; } + const root: HTMLElement = parse(dep.replace(/<\/?pre>/, '')) as any; + const links = root.querySelectorAll('a'); + const versions = new Set(); + for (const link of Array.from(links)) { + const result = extractVersionFromLinkText(link.text, packageName); + if (result) { + versions.add(result); + } + } + dependency.releases = []; + if (versions && versions.size > 0) { + dependency.releases = [...versions].map((version) => ({ + version, + })); + } + return dependency; } export async function getReleases({ diff --git a/lib/datasource/rubygems/errors.ts b/lib/datasource/rubygems/errors.ts deleted file mode 100644 index 7c9b7971d88e09..00000000000000 --- a/lib/datasource/rubygems/errors.ts +++ /dev/null @@ -1,17 +0,0 @@ -const UNAUTHORIZED = 401; -const FORBIDDEN = 403; - -const NOT_FOUND = 404; - -const REQUEST_TIMEOUT = 408; -const TOO_MANY_REQUEST = 429; -const SERVICE_UNAVAILABLE = 503; - -export { - UNAUTHORIZED, - FORBIDDEN, - NOT_FOUND, - REQUEST_TIMEOUT, - TOO_MANY_REQUEST, - SERVICE_UNAVAILABLE, -}; diff --git a/lib/datasource/rubygems/get.ts b/lib/datasource/rubygems/get.ts index 91e72767f0d136..55e1d4d2f0e51f 100644 --- a/lib/datasource/rubygems/get.ts +++ b/lib/datasource/rubygems/get.ts @@ -1,37 +1,15 @@ import { OutgoingHttpHeaders } from 'http'; import { logger } from '../../logger'; import { Http } from '../../util/http'; -import { maskToken } from '../../util/mask'; import { ensureTrailingSlash } from '../../util/url'; import { ReleaseResult } from '../common'; import { id } from './common'; -import { FORBIDDEN, NOT_FOUND, UNAUTHORIZED } from './errors'; const http = new Http(id); const INFO_PATH = '/api/v1/gems'; const VERSIONS_PATH = '/api/v1/versions'; -// istanbul ignore next -const processError = ({ err, ...rest }): null => { - const { code, statusCode, headers = {} } = err; - const data = { - ...rest, - err, - statusCode, - token: maskToken(headers.authorization) || 'none', - }; - - if (code === 'ENOTFOUND' || statusCode === NOT_FOUND) { - logger.debug(data, 'RubyGems lookup failure: not found'); - } else if (statusCode === FORBIDDEN || statusCode === UNAUTHORIZED) { - logger.debug(data, 'RubyGems lookup failure: authentication failed'); - } else { - logger.debug(data, 'RubyGems lookup failure: unknown reason'); - } - return null; -}; - const getHeaders = (): OutgoingHttpHeaders => { return { hostType: id }; }; @@ -55,47 +33,43 @@ export const getDependency = async ({ registry, }): Promise => { logger.debug({ dependency }, 'RubyGems lookup for dependency'); - try { - const info = await fetch({ dependency, registry, path: INFO_PATH }); - if (!info) { - logger.debug({ dependency }, 'RubyGems package not found.'); - return null; - } + const info = await fetch({ dependency, registry, path: INFO_PATH }); + if (!info) { + logger.debug({ dependency }, 'RubyGems package not found.'); + return null; + } - if (dependency.toLowerCase() !== info.name.toLowerCase()) { - logger.warn( - { lookup: dependency, returned: info.name }, - 'Lookup name does not match with returned.' - ); - return null; - } + if (dependency.toLowerCase() !== info.name.toLowerCase()) { + logger.warn( + { lookup: dependency, returned: info.name }, + 'Lookup name does not match with returned.' + ); + return null; + } - const versions = - (await fetch({ dependency, registry, path: VERSIONS_PATH })) || []; + const versions = + (await fetch({ dependency, registry, path: VERSIONS_PATH })) || []; - const releases = versions.map( - ({ - number: version, - platform: rubyPlatform, - created_at: releaseTimestamp, - rubygems_version: rubygemsVersion, - ruby_version: rubyVersion, - }) => ({ - version, - rubyPlatform, - releaseTimestamp, - rubygemsVersion, - rubyVersion, - }) - ); + const releases = versions.map( + ({ + number: version, + platform: rubyPlatform, + created_at: releaseTimestamp, + rubygems_version: rubygemsVersion, + ruby_version: rubyVersion, + }) => ({ + version, + rubyPlatform, + releaseTimestamp, + rubygemsVersion, + rubyVersion, + }) + ); - return { - releases, - homepage: info.homepage_uri, - sourceUrl: info.source_code_uri, - changelogUrl: info.changelog_uri, - }; - } catch (err) { - return processError({ err, registry, dependency }); - } + return { + releases, + homepage: info.homepage_uri, + sourceUrl: info.source_code_uri, + changelogUrl: info.changelog_uri, + }; }; diff --git a/lib/datasource/terraform-module/index.ts b/lib/datasource/terraform-module/index.ts index 217cdcb9827294..50c7a7b90d016e 100644 --- a/lib/datasource/terraform-module/index.ts +++ b/lib/datasource/terraform-module/index.ts @@ -101,25 +101,11 @@ export async function getReleases({ await packageCache.set(cacheNamespace, pkgUrl, dep, cacheMinutes); return dep; } catch (err) { - if (err.statusCode === 404 || err.code === 'ENOTFOUND') { - logger.debug( - { lookupName }, - `Terraform registry lookup failure: not found` - ); - logger.debug({ - err, - }); - return null; - } const failureCodes = ['EAI_AGAIN']; // istanbul ignore if if (failureCodes.includes(err.code)) { throw new ExternalHostError(err); } - logger.warn( - { err, lookupName }, - 'Terraform registry failure: Unknown error' - ); - return null; + throw err; } } diff --git a/lib/datasource/terraform-provider/index.ts b/lib/datasource/terraform-provider/index.ts index 5ba096a98e3388..64be934edfee1b 100644 --- a/lib/datasource/terraform-provider/index.ts +++ b/lib/datasource/terraform-provider/index.ts @@ -36,33 +36,22 @@ async function queryRegistry( registryURL: string, repository: string ): Promise { - try { - const backendURL = `${registryURL}/v1/providers/${repository}`; - const res = (await http.getJson(backendURL)).body; - const dep: ReleaseResult = { - name: repository, - versions: {}, - releases: null, - }; - if (res.source) { - dep.sourceUrl = res.source; - } - dep.releases = res.versions.map((version) => ({ - version, - })); - dep.homepage = `${registryURL}/providers/${repository}`; - logger.trace({ dep }, 'dep'); - return dep; - } catch (err) { - logger.debug( - { lookupName }, - `Terraform registry ("${registryURL}") lookup failure: not found` - ); - logger.debug({ - err, - }); - return null; + const backendURL = `${registryURL}/v1/providers/${repository}`; + const res = (await http.getJson(backendURL)).body; + const dep: ReleaseResult = { + name: repository, + versions: {}, + releases: null, + }; + if (res.source) { + dep.sourceUrl = res.source; } + dep.releases = res.versions.map((version) => ({ + version, + })); + dep.homepage = `${registryURL}/providers/${repository}`; + logger.trace({ dep }, 'dep'); + return dep; } async function queryReleaseBackend( @@ -72,32 +61,20 @@ async function queryReleaseBackend( ): Promise { const backendLookUpName = `terraform-provider-${lookupName}`; const backendURL = registryURL + `/index.json`; - try { - const res = ( - await http.getJson(backendURL) - ).body; - const dep: ReleaseResult = { - name: repository, - versions: {}, - releases: null, - }; - dep.releases = Object.keys(res[backendLookUpName].versions).map( - (version) => ({ - version, - }) - ); - logger.trace({ dep }, 'dep'); - return dep; - } catch (err) { - logger.debug( - { lookupName }, - `Terraform registry ("${registryURL}") lookup failure: not found` - ); - logger.debug({ - err, - }); - return null; - } + const res = (await http.getJson(backendURL)) + .body; + const dep: ReleaseResult = { + name: repository, + versions: {}, + releases: null, + }; + dep.releases = Object.keys(res[backendLookUpName].versions).map( + (version) => ({ + version, + }) + ); + logger.trace({ dep }, 'dep'); + return dep; } /**