Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: remove unnecessary catch code in datasources #6583

Merged
merged 1 commit into from Jun 25, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
9 changes: 3 additions & 6 deletions lib/datasource/cdnjs/index.ts
@@ -1,4 +1,3 @@
import { logger } from '../../logger';
import { ExternalHostError } from '../../types/errors/external-host-error';
import { Http } from '../../util/http';
import { CachePromise, cacheAble } from '../cache';
Expand Down Expand Up @@ -57,11 +56,9 @@ export async function getReleases({
}
return result;
} catch (err) {
if (err.statusCode === 404) {
logger.debug({ library }, 'cdnjs library not found');
return null;
if (err.statusCode !== 404) {
throw new ExternalHostError(err);
}
// Throw an ExternalHostError for all other types of errors
throw new ExternalHostError(err);
throw err;
}
}
9 changes: 1 addition & 8 deletions lib/datasource/crate/index.ts
@@ -1,4 +1,3 @@
import { logger } from '../../logger';
import { ExternalHostError } from '../../types/errors/external-host-error';
import * as packageCache from '../../util/cache/package';
import { Http } from '../../util/http';
Expand Down Expand Up @@ -66,18 +65,12 @@ export async function getReleases({
await packageCache.set(cacheNamespace, cacheKey, result, cacheMinutes);
return result;
} catch (err) {
if (err.statusCode === 404 || err.code === 'ENOTFOUND') {
logger.debug({ lookupName }, `Dependency lookup failure: not found`);
logger.debug({ err }, 'Crate lookup error');
return null;
}
if (
err.statusCode === 429 ||
(err.statusCode >= 500 && err.statusCode < 600)
) {
throw new ExternalHostError(err);
}
logger.warn({ err, lookupName }, 'crates.io lookup failure: Unknown error');
return null;
throw err;
}
}
12 changes: 1 addition & 11 deletions lib/datasource/dart/index.ts
@@ -1,4 +1,3 @@
import { logger } from '../../logger';
import { ExternalHostError } from '../../types/errors/external-host-error';
import { Http, HttpResponse } from '../../util/http';
import { GetReleasesConfig, ReleaseResult } from '../common';
Expand All @@ -25,22 +24,13 @@ export async function getReleases({
try {
raw = await http.getJson<DartResult>(pkgUrl);
} catch (err) {
if (err.statusCode === 404 || err.code === 'ENOTFOUND') {
logger.debug({ lookupName }, `Dependency lookup failure: not found`);
logger.debug({ err }, 'Dart lookup error');
return null;
}
if (
err.statusCode === 429 ||
(err.statusCode >= 500 && err.statusCode < 600)
) {
throw new ExternalHostError(err);
}
logger.warn(
{ err, lookupName },
'pub.dartlang.org lookup failure: Unknown error'
);
return null;
throw err;
}

const body = raw && raw.body;
Expand Down
26 changes: 1 addition & 25 deletions lib/datasource/docker/index.ts
Expand Up @@ -417,25 +417,12 @@ async function getTags(
if (err instanceof ExternalHostError) {
throw err;
}
logger.debug(
{
err,
},
'docker.getTags() error'
);
if (err.statusCode === 404 && !repository.includes('/')) {
logger.debug(
`Retrying Tags for ${registry}/${repository} using library/ prefix`
);
return getTags(registry, 'library/' + repository);
}
if (err.statusCode === 401 || err.statusCode === 403) {
logger.debug(
{ registry, dockerRepository: repository, err },
'Not authorised to look up docker tags'
);
return null;
}
// prettier-ignore
if (err.statusCode === 429 && registry.endsWith('docker.io')) { // lgtm [js/incomplete-url-substring-sanitization]
logger.warn(
Expand All @@ -451,18 +438,7 @@ async function getTags(
);
throw new ExternalHostError(err);
}
if (err.code === 'ETIMEDOUT') {
logger.debug(
{ registry },
'Timeout when attempting to connect to docker registry'
);
return null;
}
logger.warn(
{ registry, dockerRepository: repository, err },
'Error getting docker image tags'
);
return null;
throw err;
}
}

Expand Down
2 changes: 1 addition & 1 deletion lib/datasource/galaxy/index.ts
Expand Up @@ -99,6 +99,6 @@ export async function getReleases({
) {
throw new ExternalHostError(err);
}
return null;
throw err;
}
}
147 changes: 68 additions & 79 deletions lib/datasource/git-refs/index.ts
@@ -1,5 +1,4 @@
import simpleGit from 'simple-git/promise';
import { logger } from '../../logger';
import * as packageCache from '../../util/cache/package';
import * as semver from '../../versioning/semver';
import { DigestConfig, GetReleasesConfig, ReleaseResult } from '../common';
Expand All @@ -21,90 +20,80 @@ export async function getRawRefs({
lookupName,
}: GetReleasesConfig): Promise<RawRefs[] | null> {
const git = simpleGit();
try {
const cacheNamespace = 'git-raw-refs';

const cachedResult = await packageCache.get<RawRefs[]>(
cacheNamespace,
lookupName
);
/* istanbul ignore next line */
if (cachedResult) {
return cachedResult;
}

// fetch remote tags
const lsRemote = await git.listRemote([lookupName]);
if (!lsRemote) {
return null;
}

const refMatch = /(?<hash>.*?)\s+refs\/(?<type>.*?)\/(?<value>.*)/;
const headMatch = /(?<hash>.*?)\s+HEAD/;

const refs = lsRemote
.trim()
.split('\n')
.map((line) => line.trim())
.map((line) => {
let match = refMatch.exec(line);
if (match) {
return {
type: match.groups.type,
value: match.groups.value,
hash: match.groups.hash,
};
}
match = headMatch.exec(line);
if (match) {
return {
type: '',
value: 'HEAD',
hash: match.groups.hash,
};
}
// istanbul ignore next
return null;
})
.filter(Boolean)
.filter((ref) => ref.type !== 'pull' && !ref.value.endsWith('^{}'));
await packageCache.set(cacheNamespace, lookupName, refs, cacheMinutes);
return refs;
} catch (err) {
logger.info({ err }, `Git-Raw-Refs lookup error in ${lookupName}`);
const cacheNamespace = 'git-raw-refs';

const cachedResult = await packageCache.get<RawRefs[]>(
cacheNamespace,
lookupName
);
/* istanbul ignore next line */
if (cachedResult) {
return cachedResult;
}
return null;

// fetch remote tags
const lsRemote = await git.listRemote([lookupName]);
if (!lsRemote) {
return null;
}

const refMatch = /(?<hash>.*?)\s+refs\/(?<type>.*?)\/(?<value>.*)/;
const headMatch = /(?<hash>.*?)\s+HEAD/;

const refs = lsRemote
.trim()
.split('\n')
.map((line) => line.trim())
.map((line) => {
let match = refMatch.exec(line);
if (match) {
return {
type: match.groups.type,
value: match.groups.value,
hash: match.groups.hash,
};
}
match = headMatch.exec(line);
if (match) {
return {
type: '',
value: 'HEAD',
hash: match.groups.hash,
};
}
// istanbul ignore next
return null;
})
.filter(Boolean)
.filter((ref) => ref.type !== 'pull' && !ref.value.endsWith('^{}'));
await packageCache.set(cacheNamespace, lookupName, refs, cacheMinutes);
return refs;
}

export async function getReleases({
lookupName,
}: GetReleasesConfig): Promise<ReleaseResult | null> {
try {
const rawRefs: RawRefs[] = await getRawRefs({ lookupName });

const refs = rawRefs
.filter((ref) => ref.type === 'tags' || ref.type === 'heads')
.map((ref) => ref.value)
.filter((ref) => semver.isVersion(ref));

const uniqueRefs = [...new Set(refs)];

const sourceUrl = lookupName.replace(/\.git$/, '').replace(/\/$/, '');

const result: ReleaseResult = {
sourceUrl,
releases: uniqueRefs.map((ref) => ({
version: ref,
gitRef: ref,
newDigest: rawRefs.find((rawRef) => rawRef.value === ref).hash,
})),
};

return result;
} catch (err) {
logger.error({ err }, `Git-Refs lookup error in ${lookupName}`);
}
return null;
const rawRefs: RawRefs[] = await getRawRefs({ lookupName });

const refs = rawRefs
.filter((ref) => ref.type === 'tags' || ref.type === 'heads')
.map((ref) => ref.value)
.filter((ref) => semver.isVersion(ref));

const uniqueRefs = [...new Set(refs)];

const sourceUrl = lookupName.replace(/\.git$/, '').replace(/\/$/, '');

const result: ReleaseResult = {
sourceUrl,
releases: uniqueRefs.map((ref) => ({
version: ref,
gitRef: ref,
newDigest: rawRefs.find((rawRef) => rawRef.value === ref).hash,
})),
};

return result;
}

export async function getDigest(
Expand Down
42 changes: 18 additions & 24 deletions lib/datasource/git-submodules/index.ts
@@ -1,7 +1,6 @@
import { URL } from 'url';
import Git from 'simple-git/promise';

import { logger } from '../../logger';
import * as packageCache from '../../util/cache/package';
import { DigestConfig, GetReleasesConfig, ReleaseResult } from '../common';

Expand All @@ -23,31 +22,26 @@ export async function getReleases({
}

const git = Git();
try {
const newHash = (
await git.listRemote(['--refs', registryUrls[0], registryUrls[1]])
)
.trim()
.split(/\t/)[0];
const newHash = (
await git.listRemote(['--refs', registryUrls[0], registryUrls[1]])
)
.trim()
.split(/\t/)[0];

const sourceUrl = new URL(registryUrls[0]);
sourceUrl.username = '';
const sourceUrl = new URL(registryUrls[0]);
sourceUrl.username = '';

const result = {
sourceUrl: sourceUrl.href,
releases: [
{
version: newHash,
},
],
};
const cacheMinutes = 60;
await packageCache.set(cacheNamespace, cacheKey, result, cacheMinutes);
return result;
} catch (err) {
logger.debug({ err }, `Git-SubModules lookup error in ${lookupName}`);
}
return null;
const result = {
sourceUrl: sourceUrl.href,
releases: [
{
version: newHash,
},
],
};
const cacheMinutes = 60;
await packageCache.set(cacheNamespace, cacheKey, result, cacheMinutes);
return result;
}

export const getDigest = (
Expand Down
20 changes: 5 additions & 15 deletions lib/datasource/github-releases/index.ts
@@ -1,4 +1,3 @@
import { logger } from '../../logger';
import * as packageCache from '../../util/cache/package';
import { GithubHttp } from '../../util/http/github';
import { GetReleasesConfig, ReleaseResult } from '../common';
Expand Down Expand Up @@ -27,7 +26,6 @@ type GithubRelease = {
export async function getReleases({
lookupName: repo,
}: GetReleasesConfig): Promise<ReleaseResult | null> {
let githubReleases: GithubRelease[];
const cachedResult = await packageCache.get<ReleaseResult>(
cacheNamespace,
repo
Expand All @@ -36,19 +34,11 @@ export async function getReleases({
if (cachedResult) {
return cachedResult;
}
try {
const url = `https://api.github.com/repos/${repo}/releases?per_page=100`;
const res = await http.getJson<GithubRelease[]>(url, {
paginate: true,
});
githubReleases = res.body;
} catch (err) /* istanbul ignore next */ {
logger.debug({ repo, err }, 'Error retrieving from github');
}
// istanbul ignore if
if (!githubReleases) {
return null;
}
const url = `https://api.github.com/repos/${repo}/releases?per_page=100`;
const res = await http.getJson<GithubRelease[]>(url, {
paginate: true,
});
const githubReleases = res.body;
const dependency: ReleaseResult = {
sourceUrl: 'https://github.com/' + repo,
releases: null,
Expand Down