Merge branch 'master' into master

.gitignore

@@ -10,7 +10,7 @@
 /package-lock.json
 *.pyc
 renovate-0.0.0-semantic-release.tgz
-/e2e/node_modules
+/test/e2e/node_modules
 .eslintcache
 junit.xml
 /test-results

docs/development/local-development.md

@@ -142,7 +142,7 @@ You also need to make sure that you don't have a local `.npmrc` file that overri
 The Renovate project maintains 100% test coverage, so any Pull Request will fail if it does not contain full coverage for code.
 Using `// istanbul ignore` is not ideal but sometimes is a pragmatic solution if an additional test wouldn't really prove anything.
 
-To view the current test coverage locally, open up `coverage/lcov-report/index.html` in your browser.
+To view the current test coverage locally, open up `coverage/index.html` in your browser.
 
 Do not let coverage put you off submitting a PR! Maybe we can help, or at least guide.
 Also, it can be good to submit your PR as a work in progress (WIP) without tests first so that you can get a thumbs up from others about the changes, and write tests after.

docs/usage/configuration-options.md

@@ -445,6 +445,67 @@ Example for configuring `docker` auth:
 }
 ```
 
+### abortIgnoreStatusCodes
+
+This field can be used to configure status codes that Renovate ignores and passes through when `abortOnError` is set to `true`. For example to also skip 404 responses then configure the following:
+
+```json
+{
+  "hostRules": [
+    {
+      "abortOnError": true,
+      "abortStatusCodes": [404]
+    }
+  ]
+}
+```
+
+Note that this field is _not_ mergeable, so the last-applied host rule will take precedence.
+
+### abortOnError
+
+Use this field to configure Renovate to abort runs for custom hosts. By default, Renovate will only abort for known public hosts, which has the downside that transient errors for other hosts can cause autoclosing of PRs.
+
+To abort Renovate runs for http failures from _any_ host:
+
+```json
+{
+  "hostRules": [
+    {
+      "abortOnError": true
+    }
+  ]
+}
+```
+
+To abort Renovate runs for any `docker` datasource failures:
+
+```json
+{
+  "hostRules": [
+    {
+      "hostType": "docker",
+      "abortOnError": true
+    }
+  ]
+}
+```
+
+To abort Renovate for errors for a specific `docker` host:
+
+```json
+{
+  "hostRules": [
+    {
+      "hostName": "docker.company.com",
+      "abortOnError": true
+    }
+  ]
+}
+```
+
+When this field is enabled, Renovate will abort its run if it encounters either (a) any low-level http error (e.g. `ETIMEDOUT`) or (b) receives a response _not_ matching any of the configured `abortIgnoreStatusCodes` (e.g. `500 Internal Error`);
+
 ### baseUrl
 
 Use this instead of `domainName` or `hostName` if you need a rule to apply to a specific path on a host. For example, `"baseUrl": "https://api.github.com"` is equivalent to `"hostName": "api.github.com"` but `"baseUrl": "https://api.github.com/google/"` is not.

lib/config/definitions.ts

@@ -38,19 +38,24 @@ export interface RenovateOptionBase {
   stage?: RenovateConfigStage;
 }
 
-export interface RenovateArrayOption<T extends string | object = object>
-  extends RenovateOptionBase {
+export interface RenovateArrayOption<
+  T extends string | number | object = object
+> extends RenovateOptionBase {
   default?: T[];
   mergeable?: boolean;
   type: 'array';
-  subType?: 'string' | 'object';
+  subType?: 'string' | 'object' | 'number';
 }
 
 export interface RenovateStringArrayOption extends RenovateArrayOption<string> {
   format?: 'regex';
   subType: 'string';
 }
 
+export interface RenovateNumberArrayOption extends RenovateArrayOption<number> {
+  subType: 'number';
+}
+
 export interface RenovateBooleanOption extends RenovateOptionBase {
   default?: boolean;
   type: 'boolean';
@@ -79,6 +84,7 @@ export interface RenovateObjectOption extends RenovateOptionBase {
 
 export type RenovateOptions =
   | RenovateStringOption
+  | RenovateNumberArrayOption
   | RenovateStringArrayOption
   | RenovateIntegerOption
   | RenovateBooleanOption
@@ -1613,6 +1619,28 @@ const options: RenovateOptions[] = [
     cli: false,
     env: false,
   },
+  {
+    name: 'abortOnError',
+    description:
+      'If enabled, Renovate will abort its run when http request errors occur.',
+    type: 'boolean',
+    stage: 'repository',
+    parent: 'hostRules',
+    default: false,
+    cli: false,
+    env: false,
+  },
+  {
+    name: 'abortIgnoreStatusCodes',
+    description:
+      'A list of HTTP status codes to ignore and *not* abort the run because of when abortOnError=true.',
+    type: 'array',
+    subType: 'number',
+    stage: 'repository',
+    parent: 'hostRules',
+    cli: false,
+    env: false,
+  },
   {
     name: 'prBodyDefinitions',
     description: 'Table column definitions for use in PR tables',

lib/config/presets/github/index.ts

@@ -1,6 +1,6 @@
-import { PLATFORM_FAILURE } from '../../../constants/error-messages';
 import { PLATFORM_TYPE_GITHUB } from '../../../constants/platforms';
 import { logger } from '../../../logger';
+import { ExternalHostError } from '../../../types/errors/external-host-error';
 import { Http, HttpOptions } from '../../../util/http';
 import { Preset, PresetConfig } from '../common';
 import { PRESET_DEP_NOT_FOUND, fetchPreset } from '../util';
@@ -27,7 +27,7 @@ export async function fetchJSONFile(
     res = await http.getJson(url, opts);
   } catch (err) {
     // istanbul ignore if: not testable with nock
-    if (err.message === PLATFORM_FAILURE) {
+    if (err instanceof ExternalHostError) {
       throw err;
     }
     logger.debug(

lib/config/presets/gitlab/__snapshots__/index.spec.ts.snap

@@ -70,7 +70,7 @@ Array [
 ]
 `;
 
-exports[`config/presets/gitlab/index getPreset() throws platform-failure 1`] = `
+exports[`config/presets/gitlab/index getPreset() throws EXTERNAL_HOST_ERROR 1`] = `
 Array [
   Object {
     "headers": Object {

lib/config/presets/gitlab/index.spec.ts

@@ -1,6 +1,6 @@
 import * as httpMock from '../../../../test/httpMock';
 import { getName } from '../../../../test/util';
-import { PLATFORM_FAILURE } from '../../../constants/error-messages';
+import { EXTERNAL_HOST_ERROR } from '../../../constants/error-messages';
 import { PRESET_DEP_NOT_FOUND } from '../util';
 import * as gitlab from '.';
 
@@ -18,14 +18,14 @@ describe(getName(__filename), () => {
   });
 
   describe('getPreset()', () => {
-    it('throws platform-failure', async () => {
+    it('throws EXTERNAL_HOST_ERROR', async () => {
       httpMock.scope(gitlabApiHost).get(`${basePath}/branches`).reply(500);
       await expect(
         gitlab.getPreset({
           packageName: 'some/repo',
           presetName: 'non-default',
         })
-      ).rejects.toThrow(PLATFORM_FAILURE);
+      ).rejects.toThrow(EXTERNAL_HOST_ERROR);
       expect(httpMock.getTrace()).toMatchSnapshot();
     });
 

lib/config/presets/gitlab/index.ts

@@ -1,5 +1,5 @@
-import { PLATFORM_FAILURE } from '../../../constants/error-messages';
 import { logger } from '../../../logger';
+import { ExternalHostError } from '../../../types/errors/external-host-error';
 import type { GitLabBranch } from '../../../types/platform/gitlab';
 import { GitlabHttp } from '../../../util/http/gitlab';
 import { Preset, PresetConfig } from '../common';
@@ -42,7 +42,7 @@ export async function fetchJSONFile(
     const url = `${endpoint}projects/${urlEncodedRepo}/repository/files/${urlEncodedPkgName}/raw?ref=${defautlBranchName}`;
     return (await gitlabApi.getJson<Preset>(url)).body;
   } catch (err) {
-    if (err.message === PLATFORM_FAILURE) {
+    if (err instanceof ExternalHostError) {
       throw err;
     }
     logger.debug(

lib/config/presets/index.ts

@@ -1,10 +1,7 @@
 import is from '@sindresorhus/is';
-import {
-  CONFIG_VALIDATION,
-  DATASOURCE_FAILURE,
-  PLATFORM_FAILURE,
-} from '../../constants/error-messages';
+import { CONFIG_VALIDATION } from '../../constants/error-messages';
 import { logger } from '../../logger';
+import { ExternalHostError } from '../../types/errors/external-host-error';
 import { regEx } from '../../util/regex';
 import { RenovateConfig } from '../common';
 import * as massage from '../massage';
@@ -207,10 +204,7 @@ export async function resolveConfigPresets(
         } catch (err) {
           logger.debug({ preset, err }, 'Preset fetch error');
           // istanbul ignore if
-          if (
-            err.message === PLATFORM_FAILURE ||
-            err.message === DATASOURCE_FAILURE
-          ) {
+          if (err instanceof ExternalHostError) {
             throw err;
           }
           const error = new Error(CONFIG_VALIDATION);

lib/constants/error-messages.ts

@@ -5,7 +5,6 @@ export const SYSTEM_INSUFFICIENT_MEMORY = 'out-of-memory';
 // Platform Error
 export const PLATFORM_AUTHENTICATION_ERROR = 'authentication-error';
 export const PLATFORM_BAD_CREDENTIALS = 'bad-credentials';
-export const PLATFORM_FAILURE = 'platform-failure';
 export const PLATFORM_GPG_FAILED = 'gpg-failed';
 export const PLATFORM_INTEGRATION_UNAUTHORIZED = 'integration-unauthorized';
 export const PLATFORM_NOT_FOUND = 'platform-not-found';
@@ -34,8 +33,9 @@ export const REPOSITORY_UNINITIATED = 'uninitiated';
 export const MANAGER_LOCKFILE_ERROR = 'lockfile-error';
 export const MANAGER_NO_PACKAGE_FILES = 'no-package-files';
 
-// Datasource error
-export const DATASOURCE_FAILURE = 'registry-failure';
+// Host error
+export const EXTERNAL_HOST_ERROR = 'external-host-error';
+export const IGNORABLE_HOST_ERROR = 'ignorable-host-error';
 
 // Worker Error
 export const WORKER_FILE_UPDATE_FAILED = 'update-failure';

lib/datasource/cache.ts

@@ -1,5 +1,5 @@
 import { logger } from '../logger';
-import * as globalCache from '../util/cache/global';
+import * as packageCache from '../util/cache/package';
 
 /**
  * Cache callback result which has to be returned by the `CacheCallback` function.
@@ -58,7 +58,10 @@ export async function cacheAble<TArg, TResult = unknown>({
 }: CacheConfig<TArg, TResult>): Promise<TResult> {
   const cacheNamespace = `datasource-${id}`;
   const cacheKey = JSON.stringify(lookup);
-  const cachedResult = await globalCache.get<TResult>(cacheNamespace, cacheKey);
+  const cachedResult = await packageCache.get<TResult>(
+    cacheNamespace,
+    cacheKey
+  );
   // istanbul ignore if
   if (cachedResult) {
     logger.trace({ id, lookup }, 'datasource cachedResult');
@@ -69,7 +72,7 @@ export async function cacheAble<TArg, TResult = unknown>({
   if (isPrivate) {
     logger.trace({ id, lookup }, 'Skipping datasource cache for private data');
   } else {
-    await globalCache.set(cacheNamespace, cacheKey, data, minutes);
+    await packageCache.set(cacheNamespace, cacheKey, data, minutes);
   }
   return data;
 }

lib/datasource/cdnjs/index.spec.ts

@@ -1,7 +1,7 @@
 import fs from 'fs';
 import { getPkgReleases } from '..';
 import * as httpMock from '../../../test/httpMock';
-import { DATASOURCE_FAILURE } from '../../constants/error-messages';
+import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages';
 import { id as datasource } from '.';
 
 let res1 = fs.readFileSync(
@@ -36,14 +36,14 @@ describe('datasource/cdnjs', () => {
       httpMock.scope(baseUrl).get(pathFor('foo/bar')).reply(200, null);
       await expect(
         getPkgReleases({ datasource, depName: 'foo/bar' })
-      ).rejects.toThrow(DATASOURCE_FAILURE);
+      ).rejects.toThrow(EXTERNAL_HOST_ERROR);
       expect(httpMock.getTrace()).toMatchSnapshot();
     });
     it('throws for error', async () => {
       httpMock.scope(baseUrl).get(pathFor('foo/bar')).replyWithError('error');
       await expect(
         getPkgReleases({ datasource, depName: 'foo/bar' })
-      ).rejects.toThrow(DATASOURCE_FAILURE);
+      ).rejects.toThrow(EXTERNAL_HOST_ERROR);
       expect(httpMock.getTrace()).toMatchSnapshot();
     });
     it('returns null for 404', async () => {
@@ -70,28 +70,28 @@ describe('datasource/cdnjs', () => {
       httpMock.scope(baseUrl).get(pathFor('foo/bar')).reply(401);
       await expect(
         getPkgReleases({ datasource, depName: 'foo/bar' })
-      ).rejects.toThrow(DATASOURCE_FAILURE);
+      ).rejects.toThrow(EXTERNAL_HOST_ERROR);
       expect(httpMock.getTrace()).toMatchSnapshot();
     });
     it('throws for 429', async () => {
       httpMock.scope(baseUrl).get(pathFor('foo/bar')).reply(429);
       await expect(
         getPkgReleases({ datasource, depName: 'foo/bar' })
-      ).rejects.toThrow(DATASOURCE_FAILURE);
+      ).rejects.toThrow(EXTERNAL_HOST_ERROR);
       expect(httpMock.getTrace()).toMatchSnapshot();
     });
     it('throws for 5xx', async () => {
       httpMock.scope(baseUrl).get(pathFor('foo/bar')).reply(502);
       await expect(
         getPkgReleases({ datasource, depName: 'foo/bar' })
-      ).rejects.toThrow(DATASOURCE_FAILURE);
+      ).rejects.toThrow(EXTERNAL_HOST_ERROR);
       expect(httpMock.getTrace()).toMatchSnapshot();
     });
     it('returns null for unknown error', async () => {
       httpMock.scope(baseUrl).get(pathFor('foo/bar')).replyWithError('error');
       await expect(
         getPkgReleases({ datasource, depName: 'foo/bar' })
-      ).rejects.toThrow(DATASOURCE_FAILURE);
+      ).rejects.toThrow(EXTERNAL_HOST_ERROR);
       expect(httpMock.getTrace()).toMatchSnapshot();
     });
     it('processes real data', async () => {

lib/datasource/cdnjs/index.ts

@@ -1,7 +1,7 @@
-import { logger } from '../../logger';
+import { ExternalHostError } from '../../types/errors/external-host-error';
 import { Http } from '../../util/http';
 import { CachePromise, cacheAble } from '../cache';
-import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common';
+import { GetReleasesConfig, ReleaseResult } from '../common';
 
 export const id = 'cdnjs';
 
@@ -56,11 +56,9 @@ export async function getReleases({
     }
     return result;
   } catch (err) {
-    if (err.statusCode === 404) {
-      logger.debug({ library }, 'cdnjs library not found');
-      return null;
+    if (err.statusCode !== 404) {
+      throw new ExternalHostError(err);
     }
-    // Throw a DatasourceError for all other types of errors
-    throw new DatasourceError(err);
+    throw err;
   }
 }

lib/datasource/clojure/index.ts

@@ -3,5 +3,6 @@ import { MAVEN_REPO } from '../maven/common';
 export const id = 'clojure';
 
 export const defaultRegistryUrls = ['https://clojars.org/repo', MAVEN_REPO];
+export const registryStrategy = 'merge';
 
 export { getReleases } from '../maven';